Merge pull request #681 from ansible/ryansb/prebaked-conductors

Prebaked conductors, redux

.travis.yml

@@ -1,6 +1,20 @@
 language: python
 dist: trusty
 sudo: required
+env:
+  - BASE_DISTRO: 'centos:7'
+  - BASE_DISTRO: 'fedora:26'
+  - BASE_DISTRO: 'fedora:25'
+  - BASE_DISTRO: 'fedora:24'
+  - BASE_DISTRO: 'debian:jessie'
+  - BASE_DISTRO: 'debian:stretch'
+  - BASE_DISTRO: 'debian:wheezy'
+  - BASE_DISTRO: 'ubuntu:precise'
+  - BASE_DISTRO: 'ubuntu:trusty'
+  - BASE_DISTRO: 'ubuntu:xenial'
+  - BASE_DISTRO: 'ubuntu:zesty'
+  - BASE_DISTRO: 'alpine:3.5'
+  - BASE_DISTRO: 'alpine:3.4'
 
 services:
   - docker
@@ -17,20 +31,25 @@ script:
   - pip install 'ansible>=2.3'
   - ansible --version
 
-  # Install Ansible Container 
+  - pip install -U --force-reinstall -r requirements.txt
+  # Install Ansible Container
   - pip install -e .[docker]
-
   # Establish a tmp directory
   - mkdir tmp 
   - export TMPDIR=${PWD}/tmp
 
-  # run tests 
-  - python ./setup.py test
+  # run tests
+  - python ./setup.py prebake --distros $BASE_DISTRO --debug
+  - python ./setup.py test --ansible-args="-vvv"
   - if [ -f ./task.output ]; then cat task.output; fi
 
 after_success:
   - bash <(curl -s https://codecov.io/bash) -F unit        -f test/reports/unit/coverage.xml
   - bash <(curl -s https://codecov.io/bash) -F integration -f test/reports/integration/coverage.xml
 
+after_failure:
+  - docker logs `docker ps -lq`
+  - docker inspect `docker ps -lq`
+
 notifications:
   email: false

AUTHORS

@@ -1,12 +1,12 @@
 Ansible Container has been contribued to by the following authors:
 This list is automatically generated - please file an issue for corrections)
 
-Joshua "jag" Ginsberg <[email protected]>
 Chris Houseknecht <[email protected]>
+Joshua "jag" Ginsberg <[email protected]>
 Ryan Brown <[email protected]>
 Shubham Minglani <[email protected]>
-Matt Clay <[email protected]>
 Pierre-Louis Bonicoli <[email protected]>
+Matt Clay <[email protected]>
 Greg DeKoenigsberg <[email protected]>
 Marc Sensenich <[email protected]>
 Sandra Wills <[email protected]>
@@ -49,6 +49,7 @@ Alex Yanchenko <[email protected]>
 Balazs Zagyvai <[email protected]>
 Trishna Guha <[email protected]>
 Ali Asad Lotia <[email protected]>
+szinck1 <[email protected]>
 Erik Nelson <[email protected]>
 Chrrrles Paul <[email protected]>
 Andrea De Pirro <[email protected]>

CONTRIBUTORS.md

@@ -1,8 +1,8 @@
 # How to Contribute
 
-This is a very young project, and the codebase is moving quickly -- but there are plenty of ways to help. This document 
-provides some guidelines for when you're ready to ask questions, file issues, submit code, or submit examples of working Ansible 
-Container projects. 
+This is a very young project, and the codebase is moving quickly -- but there are plenty of ways to help. This document
+provides some guidelines for when you're ready to ask questions, file issues, submit code, or submit examples of working Ansible
+Container projects.
 
 For additional information about contributing to the project, or to review our *Code of Conduct* and our *Contributor's License
 Agreement*, please visit [Community Information and Contributing](https://docs.ansible.com/ansible-container/community/index.html).
@@ -17,18 +17,18 @@ When filing an issue, observing the following guidelines will lead to a quicker
 - Search existing issues to see if someone else has asked the same question or raised a similar issue.
 - State the question or nature of the issue clearly in the title.
 - When reporting a bug, please complete the issue template with as much detail as possible. The template asks for information about your environment,
-  the command you executed, stracktraces, etc. Answer each question thoroughly, and include code snippets, screen shots, or anything you feel will 
+  the command you executed, stracktraces, etc. Answer each question thoroughly, and include code snippets, screen shots, or anything you feel will
   help a developer reproduce and troubleshoot the issue.
 
 ## Contribute code
 
-Code submissions are accepted via pull requests, and they are always welcome! We may not accept them all, but we are always happy to review and 
+Code submissions are accepted via pull requests, and they are always welcome! We may not accept them all, but we are always happy to review and
 discuss them with you.
 
 Before submitting a large pull request for a new feature, we suggest joining the [Ansible Container Mailing List](https://groups.google.com/forum/#!forum/ansible-container),
 and submitting a topic to discuss the feature prior to submission.
 
-We reserve the right to reject submissions that are not a good fit for the project or not in keeping with the intended direction of the project. If you are unsure 
+We reserve the right to reject submissions that are not a good fit for the project or not in keeping with the intended direction of the project. If you are unsure
 as to whether a feature is a good fit, take the time to start a discussion on the mailing list.
 
 Please observe the following when submitting a PR:

container/cli.py

@@ -59,7 +59,7 @@ class HostCommand(object):
                           # 'purge': 'Delete all Ansible Container instances, volumes, and images',
                           # FIXME: implement status command
                           # 'status': 'Query the status of your project's containers/images',
-                          'deploy': 'Deploy your built images into production'
+                          'deploy': 'Deploy your built images into production',
                           }
 
     def subcmd_common_parsers(self, parser, subparser, cmd):
@@ -350,7 +350,6 @@ def __call__(self):
 
 host_commandline = HostCommand()
 
-
 def decode_b64json(encoded_params):
     # Using object_pairs_hook to preserve the original order of any dictionaries
     return json.loads(base64.b64decode(encoded_params).decode())

container/config.py

@@ -12,11 +12,11 @@
 import copy
 import json
 import re
-from six import add_metaclass, iteritems, PY2, string_types
+from six import add_metaclass, iteritems, PY2, string_types, text_type
 
 from collections import Mapping
+from .utils.ordereddict import ordereddict
 from ruamel import yaml
-
 import container
 
 if container.ENV == 'conductor':
@@ -46,7 +46,7 @@
 
 @add_metaclass(ABCMeta)
 class BaseAnsibleContainerConfig(Mapping):
-    _config = yaml.compat.ordereddict()
+    _config = ordereddict()
     base_path = None
     engine_list = ['docker', 'openshift', 'k8s']
 
@@ -63,7 +63,7 @@ def __init__(self, base_path, vars_files=None, engine_name=None, project_name=No
 
     @property
     def deployment_path(self):
-        dep_path = self.get('settings', yaml.compat.ordereddict()).get('deployment_output_path',
+        dep_path = self.get('settings', ordereddict()).get('deployment_output_path',
                             path.join(self.base_path, 'ansible-deployment/'))
         return path.normpath(path.abspath(path.expanduser(path.expandvars(dep_path))))
 
@@ -128,7 +128,7 @@ def set_env(self, env, config=None):
             except IOError:
                 raise AnsibleContainerNotInitializedException()
             except yaml.YAMLError as exc:
-                raise AnsibleContainerConfigException(u"Parsing container.yml - %s" % unicode(exc))
+                raise AnsibleContainerConfigException(u"Parsing container.yml - %s" % text_type(exc))
 
         self._validate_config(config)
 
@@ -140,7 +140,7 @@ def set_env(self, env, config=None):
 
         # Insure settings['pwd'] = base_path. Will be used later by conductor to resolve $PWD in volumes.
         if config.get('settings', None) is None:
-            config['settings'] = yaml.compat.ordereddict()
+            config['settings'] = ordereddict()
         config['settings']['pwd'] = self.base_path
 
         self._resolve_defaults(config)
@@ -176,7 +176,7 @@ def _resolve_defaults(self, config):
         """
         if config.get('defaults'):
             # convert config['defaults'] to an ordereddict()
-            tmp_defaults = yaml.compat.ordereddict()
+            tmp_defaults = ordereddict()
             tmp_defaults.update(copy.deepcopy(config['defaults']), relax=True)
             config['defaults'] = tmp_defaults
         defaults = config.setdefault('defaults', yaml.compat.ordereddict())
@@ -187,7 +187,7 @@ def _resolve_defaults(self, config):
                 defaults.update(self._get_variables_from_file(var_file=var_file), relax=True)
 
         logger.debug('The default type is', defaults=str(type(defaults)), config=str(type(config)))
-        if PY2 and type(defaults) == yaml.compat.ordereddict:
+        if PY2 and type(defaults) == ordereddict:
             defaults.update(self._get_environment_variables(), relax=True)
         else:
             defaults.update(self._get_environment_variables())
@@ -200,10 +200,10 @@ def _get_environment_variables():
         key is the result of removing 'AC_' from the variable name and converting the remainder to lowercase.
         For example, 'AC_DEBUG=1' becomes 'debug: 1'.
 
-        :return ruamel.yaml.compat.ordereddict
+        :return ruamel.ordereddict
         '''
         logger.debug(u'Getting environment variables...')
-        env_vars = yaml.compat.ordereddict()
+        env_vars = ordereddict()
         for var, value in ((k, v) for k, v in os.environ.items()
                            if k.startswith('AC_')):
             env_vars[var[3:].lower()] = value
@@ -215,7 +215,7 @@ def _get_variables_from_file(self, var_file):
         Looks for file relative to base_path. If not found, checks relative to base_path/ansible.
         If file extension is .yml | .yaml, parses as YAML, otherwise parses as JSON.
 
-        :return: ruamel.yaml.compat.ordereddict
+        :return: ruamel.ordereddict
         """
         abspath = path.abspath(var_file)
         if not path.exists(abspath):
@@ -229,12 +229,12 @@ def _get_variables_from_file(self, var_file):
             try:
                 config = yaml.round_trip_load(open(abspath))
             except yaml.YAMLError as exc:
-                raise AnsibleContainerConfigException(u"YAML exception: %s" % unicode(exc))
+                raise AnsibleContainerConfigException(u"YAML exception: %s" % text_type(exc))
         else:
             try:
                 config = json.load(open(abspath))
             except Exception as exc:
-                raise AnsibleContainerConfigException(u"JSON exception: %s" % unicode(exc))
+                raise AnsibleContainerConfigException(u"JSON exception: %s" % text_type(exc))
         return iteritems(config)
 
     TOP_LEVEL_WHITELIST = [
@@ -273,7 +273,7 @@ def _validate_config(self, config):
                     logger.warning("Version '1' is deprecated. Consider upgrading to version '2'.")
             else:
                 if config[top_level] is None:
-                    config[top_level] = yaml.compat.ordereddict()
+                    config[top_level] = ordereddict()
 
     def __getitem__(self, item):
         return self._config[item]
@@ -299,7 +299,7 @@ def __init__(self, container_config):
     def _process_section(self, section_value, callback=None, templar=None):
         if not templar:
             templar = self._templar
-        processed = yaml.compat.ordereddict()
+        processed = ordereddict()
         for key, value in section_value.items():
             if isinstance(value, string_types):
                 # strings can be templated
@@ -324,21 +324,21 @@ def _process_section(self, section_value, callback=None, templar=None):
     def _process_defaults(self):
         logger.debug('Processing defaults section...')
         self.defaults = self._process_section(
-            self._config.get('defaults', yaml.compat.ordereddict()),
+            self._config.get('defaults', ordereddict()),
             callback=lambda processed: self._templar.set_available_variables(
                 dict(processed)))
 
     def _process_top_level_sections(self):
         self._config['settings'] = self._config.get('settings', yaml.compat.ordereddict())
         for section in ['volumes', 'registries', 'secrets']:
             logger.debug('Processing section...', section=section)
-            setattr(self, section, dict(self._process_section(self._config.get(section, yaml.compat.ordereddict()))))
+            setattr(self, section, dict(self._process_section(self._config.get(section, ordereddict()))))
 
     def _process_services(self):
-        services = yaml.compat.ordereddict()
-        for service, service_data in self._config.get('services', yaml.compat.ordereddict()).items():
+        services = ordereddict()
+        for service, service_data in self._config.get('services', ordereddict()).items():
             logger.debug('Processing service...', service=service, service_data=service_data)
-            processed = yaml.compat.ordereddict()
+            processed = ordereddict()
             service_defaults = self.defaults.copy()
             for idx in range(len(service_data.get('volumes', []))):
                 # To mount the project directory, let users specify `$PWD` and

container/core.py

@@ -126,6 +126,26 @@ def hostcmd_init(base_path, project=None, force=False, **kwargs):
                                  **context)
         logger.info('Ansible Container initialized.')
 
+@host_only
+def hostcmd_prebake(distros, debug=False, cache=True, ignore_errors=False):
+    logger.info('Prebaking distros...', distros=distros, cache=cache)
+    engine_obj = load_engine(['BUILD_CONDUCTOR'], 'docker', os.getcwd(), {}, debug=debug)
+    from .docker.engine import PREBAKED_DISTROS
+    for distro in (distros or PREBAKED_DISTROS):
+        logger.info('Now prebaking Conductor image for %s', distro)
+        try:
+            engine_obj.build_conductor_image(os.getcwd(),
+                                             distro,
+                                             prebaking=True,
+                                             cache=cache)
+        except Exception as e:
+            logger.exception('Failure building prebaked image for %s', distro)
+            if ignore_errors:
+                continue
+        except KeyboardInterrupt:
+            if ignore_errors:
+                continue
+
 
 @host_only
 def hostcmd_build(base_path, project_name, engine_name, vars_files=None, **kwargs):
@@ -768,13 +788,14 @@ def conductorcmd_build(engine_name, project_name, services, cache=True, local_py
                         # No /lib volume
                         pass
                     run_kwargs['environment'].update(dict(
-                         LD_LIBRARY_PATH='/_usr/lib:/_usr/lib64:/_usr/local/lib{}'.format(extra_library_paths),
-                         CPATH='/_usr/include:/_usr/local/include',
+                         LD_LIBRARY_PATH='/usr/lib:/usr/lib64:/_usr/lib:/_usr/lib64:/_usr/local/lib{}'.format(extra_library_paths),
+                         CPATH='/usr/include:/usr/local/include:/_usr/include:/_usr/local/include',
                          PATH='/usr/local/sbin:/usr/local/bin:'
                               '/usr/sbin:/usr/bin:/sbin:/bin:'
                               '/_usr/sbin:/_usr/bin:'
                               '/_usr/local/sbin:/_usr/local/bin',
-                         PYTHONPATH='/_usr/lib/python2.7'))
+                         # PYTHONPATH='/_usr/lib/python2.7'
+                    ))
 
                 container_id = engine.run_container(cur_image_id, service_name, **run_kwargs)