diff --git a/atomicapp/cli/main.py b/atomicapp/cli/main.py index 1bb3b706..1dabbbff 100644 --- a/atomicapp/cli/main.py +++ b/atomicapp/cli/main.py @@ -64,7 +64,8 @@ def cli_fetch(args): nm = NuleculeManager(app_spec=argdict['app_spec'], destination=destination, cli_answers=argdict['cli_answers'], - answers_file=argdict['answers']) + answers_file=argdict['answers'], + answers_format=argdict.get('answers_format')) nm.fetch(**argdict) # Clean up the files if the user asked us to. Otherwise # notify the user where they can manage the application @@ -81,7 +82,8 @@ def cli_run(args): nm = NuleculeManager(app_spec=argdict['app_spec'], destination=destination, cli_answers=argdict['cli_answers'], - answers_file=argdict['answers']) + answers_file=argdict['answers'], + answers_format=argdict.get('answers_format')) nm.run(**argdict) # Clean up the files if the user asked us to. Otherwise # notify the user where they can manage the application @@ -306,7 +308,7 @@ def create_parser(self): help="A file which will contain anwsers provided in interactive mode") run_subparser.add_argument( "--provider", - dest="cli_provider", + dest="provider", choices=PROVIDERS, help="The provider to use. Overrides provider value in answerfile.") run_subparser.add_argument( @@ -511,7 +513,8 @@ def run(self): # and make a dictionary of it to pass along in args. setattr(args, 'cli_answers', {}) for item in ['provider-api', 'provider-cafile', 'provider-auth', - 'provider-config', 'provider-tlsverify', 'namespace']: + 'provider-config', 'provider-tlsverify', 'namespace', + 'provider']: if hasattr(args, item) and getattr(args, item) is not None: args.cli_answers[item] = getattr(args, item) diff --git a/atomicapp/nulecule/base.py b/atomicapp/nulecule/base.py index 3332ef5b..d0d76347 100644 --- a/atomicapp/nulecule/base.py +++ b/atomicapp/nulecule/base.py @@ -176,9 +176,8 @@ def load_from_path(cls, src, config=None, namespace=GLOBAL_CONF, raise NuleculeException("Failure parsing %s file. Validation error on line %s, column %s:\n%s" % (nulecule_path, line, column, output)) - nulecule = Nulecule(config=config, - basepath=src, namespace=namespace, - **nulecule_data) + nulecule = Nulecule(config=config, basepath=src, + namespace=namespace, **nulecule_data) nulecule.load_components(nodeps, dryrun) return nulecule @@ -247,7 +246,7 @@ def load_config(self, config=None, ask=False, skip_asking=False): # FIXME: Find a better way to expose config data to components. # A component should not get access to all the variables, # but only to variables it needs. - component.load_config(config=config.clone(component.namespace), + component.load_config(config=config, ask=ask, skip_asking=skip_asking) def load_components(self, nodeps=False, dryrun=False): @@ -294,6 +293,18 @@ def render(self, provider_key=None, dryrun=False): component.render(provider_key=provider_key, dryrun=dryrun) def _get_component_namespace(self, component_name): + """ + Get a unique namespace for a Nulecule graph item, by concatinating + the namespace of the current Nulecule (which could be the root Nulecule + app or a child or external Nulecule app) and name of the Nulecule + graph item. + + Args: + component_name (str): Name of the Nulecule graph item + + Returns: + A string + """ current_namespace = '' if self.namespace == GLOBAL_CONF else self.namespace return ( '%s%s%s' % (current_namespace, NAMESPACE_SEPARATOR, component_name) @@ -366,7 +377,7 @@ def load_config(self, config=None, ask=False, skip_asking=False): super(NuleculeComponent, self).load_config( config, ask=ask, skip_asking=skip_asking) if isinstance(self._app, Nulecule): - self._app.load_config(config=self.config.clone(self.namespace), + self._app.load_config(config=self.config, ask=ask, skip_asking=skip_asking) def load_external_application(self, dryrun=False, update=False): @@ -443,7 +454,7 @@ def render(self, provider_key=None, dryrun=False): raise NuleculeException( "Data for provider \"%s\" are not part of this app" % provider_key) - context = self.get_context() + context = self.config.context(self.namespace) for provider in self.artifacts: if provider_key and provider != provider_key: continue diff --git a/atomicapp/nulecule/config.py b/atomicapp/nulecule/config.py index f496ce1d..1bc0600a 100644 --- a/atomicapp/nulecule/config.py +++ b/atomicapp/nulecule/config.py @@ -4,8 +4,7 @@ from atomicapp.constants import (GLOBAL_CONF, LOGGER_COCKPIT, DEFAULT_PROVIDER, - DEFAULT_ANSWERS, - NAMESPACE_SEPARATOR) + DEFAULT_ANSWERS) from collections import defaultdict cockpit_logger = logging.getLogger(LOGGER_COCKPIT) @@ -13,121 +12,125 @@ class Config(object): """ - Store config data for a Nulecule or Nulecule component. - - It stores config data from different sources (answers, cli, user data) - separately, and exposes high level interfaces to read, write data - from it, with enforced read/write policies. + This class allows to store config data in different scopes along with + source info for the data. When fetching the value for a key in a scope, + the source info and the PRIORITY order of sources is taken into account. + + Data sources: + cli: Config data coming from the CLI + runtime: Config data resolved during atomic app runtime. For example, + when the value for a parameter in a Nulecule or Nulecule graph + item is missing in answers data, we first try to load the default + value for the parameter. When there's no default value, or when + the user has specified to forcefully ask the user for values, we + ask the user for data. These data collected/resolved during runtime + form the runtime data. + answers: Config data coming from answers file + defaults: Default config data specified in atomicapp/constants.py + + The priority order of the data sources is: + cli > runtime > answers > defaults """ - def __init__(self, namespace='', answers=None, cli=None, - data=None, is_nulecule=False): - self._namespace = namespace - self._is_nulecule = is_nulecule or False - self._parent_ns, self._current_ns = self._split_namespace(self._namespace) - # Store answers data - self._answers = defaultdict(dict) - self._answers.update(answers or {}) - # Store CLI data - self._cli = cli or {} - # Store data collected during runtime - self._data = data or defaultdict(dict) - self._context = None - self._provider = None - - @property - def globals(self): - """ - Get global config params dict for a Nulecule. - """ - d = self._answers.get(GLOBAL_CONF, {}) - d.update(self._data.get(GLOBAL_CONF, {})) - d.update(self._cli.get(GLOBAL_CONF, {})) - return d + PRIORITY = ( + 'cli', + 'runtime', + 'answers', + 'defaults' + ) - @property - def provider(self): + def __init__(self, answers=None, cli=None): """ - Get provider name. - - Returns: - Provider name (str) - """ - if self._provider is None: - self._provider = self._data[GLOBAL_CONF].get('provider') or \ - self._answers[GLOBAL_CONF].get('provider') - if self._provider is None: - self._data[GLOBAL_CONF]['provider'] = DEFAULT_PROVIDER - self._provider = DEFAULT_PROVIDER - - return self._provider + Initialize a Config instance. - @property - def providerconfig(self): - """ - Get provider config info taking into account answers and cli data. - """ - pass + Args: + answers (dict): Answers data + cli (dict): CLI data + """ + answers = answers or {} + cli = cli or {} + # We use a defaultdict of defaultdicts so that we can avoid doing + # redundant checks in a nested dictionary if the value of the keys + # are dictionaries or None. + self._data = defaultdict(defaultdict) + # Initialize default data dict + self._data['defaults'] = defaultdict(defaultdict) + # Initialize answers data dict + self._data['answers'] = defaultdict(defaultdict) + # Initialize cli data dict + self._data['cli'] = defaultdict(defaultdict) + # Initialize runtime data dict + self._data['runtime'] = defaultdict(defaultdict) + + # Load default answers + for scope, data in DEFAULT_ANSWERS.items(): + for key, value in data.items(): + self.set(key, value, scope=scope, source='defaults') + self.set('provider', DEFAULT_PROVIDER, scope=GLOBAL_CONF, source='defaults') + + # Load answers data + for scope, data in answers.items(): + for key, value in data.items(): + self.set(key, value, scope=scope, source='answers') + + # Load cli data + for key, value in cli.items(): + self.set(key, value, scope=GLOBAL_CONF, source='cli') + + def get(self, key, scope=GLOBAL_CONF, ignore_sources=[]): + """ + Get the value of a key in a scope. This takes care of resolving + the value by going through the PRIORITY order of the various + sources of data. - @property - def namespace(self): - """ - Get normalized namespace for this instance. + Args: + key (str): Key + scope (str): Scope from which to fetch the value for the key Returns: - Current namespace (str). + Value for the key. """ - return self._namespace or GLOBAL_CONF + for source in self.PRIORITY: + if source in ignore_sources: + continue + value = self._data[source][scope].get(key) or self._data[source][ + GLOBAL_CONF].get(key) + if value: + return value + return None - def set(self, key, value): + def set(self, key, value, source, scope=GLOBAL_CONF): """ - Set value for a key in the current namespace. + Set the value for a key within a scope along with specifying the + source of the value. Args: key (str): Key - value (str): Value. + value: Value + scope (str): Scope in which to store the value + source (str): Source of the value """ - self._data[self.namespace][key] = value + self._data[source][scope][key] = value - def get(self, key): + def context(self, scope=GLOBAL_CONF): """ - Get value for a key from data accessible from the current namespace. - - TODO: Improved data inheritance model. It makes sense for a component - to be able to access data from it's sibling namespaces and children - namespaces. + Get context data for the scope of Nulecule graph item by aggregating + the data from various sources taking their priority order into + account. This context data, which is a flat dictionary, is used to + render the variables in the artifacts of Nulecule graph item. Args: - key (str): Key - + scope (str): Scope (or namespace) for the Nulecule graph item. Returns: - Value for the key, else None. + A dictionary """ - return ( - self._data[self.namespace].get(key) or - (self._data[self._parent_ns].get(key) if self._parent_ns else None) or - self._data[GLOBAL_CONF].get(key) or - self._answers[self.namespace].get(key) or - (self._answers[self._parent_ns].get(key) if self._parent_ns else None) or - self._answers[GLOBAL_CONF].get(key) - ) - - def context(self): - """ - Get context to render artifact files in a Nulecule component. - - TODO: Improved data inheritance model. Data from siblings and children - namespaces should be available in the context to render an artifact - file in the current namespace. - """ - if self._context is None: - self._context = {} - self._context.update(copy.copy(self._data[GLOBAL_CONF])) - self._context.update(copy.copy(self._data[self.namespace])) - - self._context.update(copy.copy(self._answers[GLOBAL_CONF])) - self._context.update(copy.copy(self._answers[self.namespace])) - return self._context + result = {} + for source in reversed(self.PRIORITY): + source_data = self._data[source] + result.update(copy.deepcopy(source_data.get(GLOBAL_CONF) or {})) + if scope != GLOBAL_CONF: + result.update(copy.deepcopy(source_data.get(scope) or {})) + return result def runtime_answers(self): """ @@ -137,56 +140,34 @@ def runtime_answers(self): A defaultdict containing runtime answers data. """ answers = defaultdict(dict) - answers.update(copy.deepcopy(DEFAULT_ANSWERS)) - answers['general']['provider'] = self.provider - - for key, value in self._answers.items(): - answers[key].update(value) - for key, value in self._data.items(): - answers[key].update(value) + for source in reversed(self.PRIORITY): + for scope, data in (self._data.get(source) or {}).items(): + answers[scope].update(copy.deepcopy(data)) # Remove empty sections for answers for key, value in answers.items(): if value is None: - answers.pop(key, None) + answers.pop(key) return answers - def clone(self, namespace): + def update_source(self, source, data): """ - Create a new config instance in the specified namespace. + Update answers data for a source. Args: - name (str): Name of the child component - - Returns: - A Config instance. + source (str): Source name + data (dict): Answers data """ - config = Config(namespace=namespace, - answers=self._answers, - cli=self._cli, - data=self._data) - return config + data = data or {} + if source not in self._data: + raise - def _split_namespace(self, namespace): - """ - Split namespace to get parent and current namespace in a Nulecule. - """ - if self._is_nulecule: - return '', namespace - words = namespace.rsplit(NAMESPACE_SEPARATOR, 1) - parent, current = '', '' - if len(words) == 2: - parent, current = words[0], words[1] - else: - parent, current = '', words[0] - return parent, current - - def __eq__(self, obj): - """ - Check equality of config instances. - """ - if self._namespace == obj._namespace or self._answers == obj._answers or self._data == obj._data or self._cli == obj._cli: - return True - return False + # clean up source data + for k in self._data[source]: + self._data[source].pop(k) + + for scope, data in data.items(): + for key, value in data.items(): + self.set(key, value, scope=scope, source=source) diff --git a/atomicapp/nulecule/lib.py b/atomicapp/nulecule/lib.py index 0a82aeeb..6157bda5 100644 --- a/atomicapp/nulecule/lib.py +++ b/atomicapp/nulecule/lib.py @@ -19,7 +19,8 @@ """ import logging -from atomicapp.constants import (LOGGER_COCKPIT, +from atomicapp.constants import (GLOBAL_CONF, + LOGGER_COCKPIT, NAME_KEY, DEFAULTNAME_KEY, PROVIDERS) @@ -61,22 +62,20 @@ def load_config(self, config, ask=False, skip_asking=False): Returns: None """ - for param in self.params: - value = config.get(param[NAME_KEY]) - if value is None and (ask or ( - not skip_asking and param.get(DEFAULTNAME_KEY) is None)): - cockpit_logger.info("%s is missing in answers.conf." % param[NAME_KEY]) - value = Utils.askFor(param[NAME_KEY], param, self.namespace) - elif value is None: - value = param.get(DEFAULTNAME_KEY) - config.set(param[NAME_KEY], value) self.config = config - - def get_context(self): - """ - Get context data from config data for rendering an artifact. - """ - return self.config.context() + for param in self.params: + value = config.get(param[NAME_KEY], scope=self.namespace, ignore_sources=['defaults']) + if value is None: + if ask or (not skip_asking and + param.get(DEFAULTNAME_KEY) is None): + cockpit_logger.info( + "%s is missing in answers.conf." % param[NAME_KEY]) + value = config.get(param[NAME_KEY], scope=self.namespace) \ + or Utils.askFor(param[NAME_KEY], param, self.namespace) + else: + value = param.get(DEFAULTNAME_KEY) + config.set(param[NAME_KEY], value, source='runtime', + scope=self.namespace) def get_provider(self, provider_key=None, dry=False): """ @@ -91,7 +90,7 @@ def get_provider(self, provider_key=None, dry=False): """ # If provider_key isn't provided via CLI, let's grab it the configuration if provider_key is None: - provider_key = self.config.provider + provider_key = self.config.get('provider', scope=GLOBAL_CONF) provider_class = self.plugin.getProvider(provider_key) if provider_class is None: raise NuleculeException("Invalid Provider - '{}', provided in " @@ -99,7 +98,7 @@ def get_provider(self, provider_key=None, dry=False): .format(provider_key, ', ' .join(PROVIDERS))) return provider_key, provider_class( - self.get_context(), self.basepath, dry) + self.config.context(), self.basepath, dry) def run(self, provider_key=None, dry=False): raise NotImplementedError diff --git a/atomicapp/nulecule/main.py b/atomicapp/nulecule/main.py index a63ff94e..cf3e4719 100644 --- a/atomicapp/nulecule/main.py +++ b/atomicapp/nulecule/main.py @@ -18,7 +18,6 @@ along with Atomic App. If not, see . """ import anymarkup -import copy import distutils.dir_util import logging import os @@ -27,12 +26,10 @@ import urllib from string import Template -from atomicapp.constants import (GLOBAL_CONF, - ANSWERS_FILE_SAMPLE_FORMAT, +from atomicapp.constants import (ANSWERS_FILE_SAMPLE_FORMAT, ANSWERS_FILE, ANSWERS_FILE_SAMPLE, ANSWERS_RUNTIME_FILE, - DEFAULT_ANSWERS, LOGGER_COCKPIT, LOGGER_DEFAULT, MAIN_FILE, @@ -54,7 +51,8 @@ class NuleculeManager(object): """ def __init__(self, app_spec, destination=None, - cli_answers=None, answers_file=None): + cli_answers=None, answers_file=None, + answers_format=None): """ init function for NuleculeManager. Sets a few instance variables. @@ -64,11 +62,9 @@ def __init__(self, app_spec, destination=None, destination: where to unpack a nulecule to if it isn't local cli_answers: some answer file values provided from cli args answers_file: the location of the answers file - cli (dict): CLI data + answers_format (str): File format for writing sample answers file """ - self.answers = copy.deepcopy(DEFAULT_ANSWERS) - self.cli_answers = cli_answers - self.answers_format = None + self.answers_format = answers_format or ANSWERS_FILE_SAMPLE_FORMAT self.answers_file = None # The path to an answer file self.app_path = None # The path where the app resides or will reside self.image = None # The container image to pull the app from @@ -119,7 +115,7 @@ def __init__(self, app_spec, destination=None, # Process answers. self.answers_file = answers_file - self._process_answers() + self.config = Config(cli=cli_answers) @staticmethod def init(app_name, destination=None, app_version='1.0', @@ -221,27 +217,24 @@ def unpack(self, update=False, return Nulecule.load_from_path( self.app_path, dryrun=dryrun, config=config) - def genanswers(self, dryrun=False, answers_format=None, **kwargs): + def genanswers(self, dryrun=False, **kwargs): """ Renders artifacts and then generates an answer file. Finally copies answer file to the current working directory. Args: dryrun (bool): Do not make any change to the host system if True - answers_format (str): File format for writing sample answers file kwargs (dict): Extra keyword arguments Returns: None """ - self.answers_format = answers_format or ANSWERS_FILE_SAMPLE_FORMAT # Check to make sure an answers.conf file doesn't exist already answers_file = os.path.join(os.getcwd(), ANSWERS_FILE) if os.path.exists(answers_file): raise NuleculeException( "Can't generate answers.conf over existing file") - self.config = Config(namespace=GLOBAL_CONF) # Call unpack to get the app code self.nulecule = self.unpack(update=False, dryrun=dryrun, config=self.config) @@ -252,8 +245,7 @@ def genanswers(self, dryrun=False, answers_format=None, **kwargs): self.nulecule.config, None) self._write_answers(answers_file, answers, self.answers_format) - def fetch(self, nodeps=False, update=False, dryrun=False, - answers_format=ANSWERS_FILE_SAMPLE_FORMAT, **kwargs): + def fetch(self, nodeps=False, update=False, dryrun=False, **kwargs): """ Installs (unpacks) a Nulecule application from a Nulecule image to a target path. @@ -264,13 +256,10 @@ def fetch(self, nodeps=False, update=False, dryrun=False, update (bool): Pull requisite Nulecule image and install or update already installed Nulecule application dryrun (bool): Do not make any change to the host system if True - answers_format (str): File format for writing sample answers file kwargs (dict): Extra keyword arguments Returns: None """ - self.answers_format = answers_format or ANSWERS_FILE_SAMPLE_FORMAT - # Call unpack. If the app doesn't exist it will be pulled. If # it does exist it will be just be loaded and returned self.nulecule = self.unpack(update, dryrun, config=self.config) @@ -281,49 +270,41 @@ def fetch(self, nodeps=False, update=False, dryrun=False, # write sample answers file self._write_answers( os.path.join(self.app_path, ANSWERS_FILE_SAMPLE), - runtime_answers, answers_format) + runtime_answers, self.answers_format) cockpit_logger.info("Install Successful.") - def run(self, cli_provider, answers_output, ask, - answers_format=ANSWERS_FILE_SAMPLE_FORMAT, **kwargs): + def run(self, answers_output, ask, **kwargs): """ Runs a Nulecule application from a local path or a Nulecule image name. Args: answers (dict or str): Answers data or local path to answers file - cli_provider (str): Provider to use to run the Nulecule - application answers_output (str): Path to file to export runtime answers data to ask (bool): Ask for values for params with default values from user, if True - answers_format (str): File format for writing sample answers file kwargs (dict): Extra keyword arguments Returns: None """ - self.answers_format = answers_format or ANSWERS_FILE_SAMPLE_FORMAT dryrun = kwargs.get('dryrun') or False - # If we didn't find an answers file before then call _process_answers - # again just in case the app developer embedded an answers file - if not self.answers_file: - self._process_answers() - # Call unpack. If the app doesn't exist it will be pulled. If # it does exist it will be just be loaded and returned self.nulecule = self.unpack(dryrun=dryrun, config=self.config) + # Process answers file + self._process_answers() + self.nulecule.load_config(ask=ask) - if cli_provider: - self.nulecule.config.set('provider', cli_provider) - self.nulecule.render(cli_provider, dryrun) - self.nulecule.run(cli_provider, dryrun) + provider = self.nulecule.config.get('provider') + self.nulecule.render(provider, dryrun) + self.nulecule.run(provider, dryrun) runtime_answers = self._get_runtime_answers( - self.nulecule.config, cli_provider) + self.nulecule.config, provider) self._write_answers( os.path.join(self.app_path, ANSWERS_RUNTIME_FILE), runtime_answers, self.answers_format) @@ -331,12 +312,11 @@ def run(self, cli_provider, answers_output, ask, self._write_answers(answers_output, runtime_answers, self.answers_format) - def stop(self, cli_provider, **kwargs): + def stop(self, **kwargs): """ Stops a running Nulecule application. Args: - cli_provider (str): Provider running the Nulecule application kwargs (dict): Extra keyword arguments """ # For stop we use the generated answer file from the run @@ -347,10 +327,9 @@ def stop(self, cli_provider, **kwargs): self.nulecule = Nulecule.load_from_path( self.app_path, config=self.config, dryrun=dryrun) self.nulecule.load_config() - if cli_provider: - self.nulecule.config.set('provider', cli_provider) - self.nulecule.render(self.nulecule.config.provider, dryrun=dryrun) - self.nulecule.stop(self.nulecule.config.provider, dryrun) + self.nulecule.render(self.nulecule.config.get('provider'), + dryrun=dryrun) + self.nulecule.stop(self.nulecule.config.get('provider'), dryrun) def clean(self, force=False): # For future use @@ -372,6 +351,7 @@ def _process_answers(self): Returns: None """ + answers = None app_path_answers = os.path.join(self.app_path, ANSWERS_FILE) # If the user didn't provide an answers file then check the app @@ -398,15 +378,9 @@ def _process_answers(self): "Provided answers file doesn't exist: {}".format(self.answers_file)) # Load answers - self.answers = Utils.loadAnswers(self.answers_file) - - self.config = Config(namespace=GLOBAL_CONF, answers=self.answers, - cli={GLOBAL_CONF: self.cli_answers}) + answers = Utils.loadAnswers(self.answers_file, self.answers_format) - # If there is answers data from the cli then merge it in now - # if self.cli_answers: - # for k, v in self.cli_answers.iteritems(): - # self.answers[GLOBAL_CONF][k] = v + self.config.update_source(source='answers', data=answers) def _write_answers(self, path, answers, answers_format): """ diff --git a/atomicapp/utils.py b/atomicapp/utils.py index 8aeddb10..f1ce1243 100644 --- a/atomicapp/utils.py +++ b/atomicapp/utils.py @@ -360,13 +360,17 @@ def getUniqueUUID(): return data @staticmethod - def loadAnswers(answers_file): + def loadAnswers(answers_file, format=None): if not os.path.isfile(answers_file): raise AtomicAppUtilsException( "Provided answers file does not exist: %s" % answers_file) logger.debug("Loading answers from file: %s", answers_file) - return anymarkup.parse_file(answers_file) + try: + result = anymarkup.parse_file(answers_file, format=format) + except anymarkup.AnyMarkupError: + result = anymarkup.parse_file(answers_file) + return result @staticmethod def copy_dir(src, dest, update=False, dryrun=False):