From 19c2488d9cf8c65f2b7258a33bdfce7d4e74d1b2 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 26 Dec 2014 11:10:42 -0500 Subject: [PATCH 1/4] Refactor building of services, and add ServiceLink in preparation for adding includes. Signed-off-by: Daniel Nephin --- CONTRIBUTING.md | 2 +- fig/container.py | 1 + fig/project.py | 106 +++++++++++++++--------------- fig/service.py | 24 +++++-- tests/integration/cli_test.py | 4 +- tests/integration/project_test.py | 6 +- tests/unit/project_test.py | 78 ++++++++++++++-------- 7 files changed, 132 insertions(+), 89 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index eb101955488..b72f89bcf3a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -11,7 +11,7 @@ Pull requests will need: ## Development environment -If you're looking contribute to [Fig](http://www.fig.sh/) +If you're looking to contribute to [Fig](http://www.fig.sh/) but you're new to the project or maybe even to Python, here are the steps that should get you started. diff --git a/fig/container.py b/fig/container.py index 0ab75512062..809f2193019 100644 --- a/fig/container.py +++ b/fig/container.py @@ -148,6 +148,7 @@ def inspect(self): self.has_been_inspected = True return self.dictionary + # TODO: this is only used by tests, should move to a module under tests/ def links(self): links = [] for container in self.client.containers(): diff --git a/fig/project.py b/fig/project.py index e013da4e980..3634fcdd251 100644 --- a/fig/project.py +++ b/fig/project.py @@ -97,51 +97,57 @@ def get_services(self, service_names=None, include_links=False): Raises NoSuchService if any of the named services do not exist. """ - if service_names is None or len(service_names) == 0: - return self.get_services( - service_names=[s.name for s in self.services], - include_links=include_links - ) - else: - unsorted = [self.get_service(name) for name in service_names] - services = [s for s in self.services if s in unsorted] - - if include_links: - services = reduce(self._inject_links, services, []) - - uniques = [] - [uniques.append(s) for s in services if s not in uniques] - return uniques - - def get_links(self, service_dict): - links = [] - if 'links' in service_dict: - for link in service_dict.get('links', []): - if ':' in link: - service_name, link_name = link.split(':', 1) - else: - service_name, link_name = link, None - try: - links.append((self.get_service(service_name), link_name)) - except NoSuchService: - raise ConfigurationError('Service "%s" has a link to service "%s" which does not exist.' % (service_dict['name'], service_name)) - del service_dict['links'] - return links + + def _add_linked_services(service): + linked_services = service.get_linked_services() + if not linked_services: + return [service] + + return flat_map(_add_linked_services, linked_services) + [service] + + if not service_names: + return self.all_services + + services = [self.get_service(name) for name in service_names] + if include_links: + services = flat_map(_add_linked_services, services) + + # TODO: use orderedset/ordereddict + uniques = [] + [uniques.append(s) for s in services if s not in uniques] + return uniques + + def get_links(self, config_links, name): + def get_linked_service(link): + if ':' in link: + service_name, link_name = link.split(':', 1) + else: + service_name, link_name = link, None + + try: + return ServiceLink(self.get_service(service_name), link_name) + except NoSuchService: + raise ConfigurationError( + 'Service "%s" has a link to service "%s" which does not ' + 'exist.' % (name, service_name)) + + return map(get_linked_service, config_links or []) def get_volumes_from(self, service_dict): volumes_from = [] - if 'volumes_from' in service_dict: - for volume_name in service_dict.get('volumes_from', []): + for volume_name in service_dict.pop('volumes_from', []): + try: + service = self.get_service(volume_name) + volumes_from.append(service) + except NoSuchService: try: - service = self.get_service(volume_name) - volumes_from.append(service) - except NoSuchService: - try: - container = Container.from_id(self.client, volume_name) - volumes_from.append(container) - except APIError: - raise ConfigurationError('Service "%s" mounts volumes from "%s", which is not the name of a service or container.' % (service_dict['name'], volume_name)) - del service_dict['volumes_from'] + container = Container.from_id(self.client, volume_name) + volumes_from.append(container) + except APIError: + raise ConfigurationError( + 'Service "%s" mounts volumes from "%s", which is not ' + 'the name of a service or container.' % ( + service_dict['name'], volume_name)) return volumes_from def start(self, service_names=None, **options): @@ -205,19 +211,15 @@ def containers(self, service_names=None, stopped=False, one_off=False): for service in self.get_services(service_names) if service.has_container(container, one_off=one_off)] - def _inject_links(self, acc, service): - linked_names = service.get_linked_names() + def __repr__(self): + return "Project(%s, services=%s, includes=%s)" % ( + self.name, + len(self.services), + len(self.external_projects)) - if len(linked_names) > 0: - linked_services = self.get_services( - service_names=linked_names, - include_links=True - ) - else: - linked_services = [] - linked_services.append(service) - return acc + linked_services +def flat_map(func, seq): + return list(chain.from_iterable(map(func, seq))) class NoSuchService(Exception): diff --git a/fig/service.py b/fig/service.py index d06d271f682..467ca2b5b86 100644 --- a/fig/service.py +++ b/fig/service.py @@ -1,16 +1,18 @@ from __future__ import unicode_literals from __future__ import absolute_import + from collections import namedtuple import logging -import re -import os from operator import attrgetter +import os +import re import sys from docker.errors import APIError -from .container import Container -from .progress_stream import stream_output, StreamOutputError +from fig.container import Container +from fig.progress_stream import stream_output, StreamOutputError + log = logging.getLogger(__name__) @@ -84,6 +86,9 @@ class ConfigError(ValueError): ServiceName = namedtuple('ServiceName', 'project service number') +ServiceLink = namedtuple('ServiceLink', 'service alias') + + class Service(object): def __init__(self, name, client=None, project='default', links=None, volumes_from=None, **options): if not re.match('^%s+$' % VALID_NAME_CHARS, name): @@ -352,7 +357,10 @@ def start_or_create_containers( return [self.start_container_if_stopped(c) for c in containers] def get_linked_names(self): - return [s.name for (s, _) in self.links] + return [link.service.full_name for link in self.links] + + def get_linked_services(self): + return [link.service for link in self.links] def _next_container_name(self, all_containers, one_off=False): bits = [self.project, self.name] @@ -508,6 +516,12 @@ def pull(self, insecure_registry=False): insecure_registry=insecure_registry ) + def __repr__(self): + return "Service(%s, project='%s', links=%r)" % ( + self.name, + self.project, + self.get_linked_names()) + NAME_RE = re.compile(r'^([^_]+)_([^_]+)_(run_)?(\d+)$') diff --git a/tests/integration/cli_test.py b/tests/integration/cli_test.py index 2f7ecb59492..21e025708cb 100644 --- a/tests/integration/cli_test.py +++ b/tests/integration/cli_test.py @@ -1,8 +1,10 @@ from __future__ import absolute_import +import contextlib +import os import sys -from six import StringIO from mock import patch +from six import StringIO from .testcases import DockerClientTestCase from fig.cli.main import TopLevelCommand diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index ce087245813..1271cc189d0 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -1,6 +1,8 @@ from __future__ import unicode_literals -from fig.project import Project, ConfigurationError + from fig.container import Container +from fig.project import Project, ConfigurationError +from fig.service import ServiceLink from .testcases import DockerClientTestCase @@ -184,7 +186,7 @@ def test_project_up_without_all_services(self): def test_project_up_starts_links(self): console = self.create_service('console') db = self.create_service('db', volumes=['/var/db']) - web = self.create_service('web', links=[(db, 'db')]) + web = self.create_service('web', links=[ServiceLink(db, 'db')]) project = Project('figtest', [web, db, console], self.client) project.start() diff --git a/tests/unit/project_test.py b/tests/unit/project_test.py index 5c8d35b1d92..d6c6c44677a 100644 --- a/tests/unit/project_test.py +++ b/tests/unit/project_test.py @@ -65,7 +65,7 @@ def test_from_config_throws_error_when_not_dict(self): 'web': 'busybox:latest', }, None) - def test_get_service(self): + def test_get_service_no_external(self): web = Service( project='figtest', name='web', @@ -75,6 +75,24 @@ def test_get_service(self): project = Project('test', [web], None) self.assertEqual(project.get_service('web'), web) + def test_get_service_with_project_name(self): + web = Service( project='figtest', name='web') + project = Project('test', [web], None, None) + self.assertEqual(project.get_service('test_web'), web) + + def test_get_service_not_found(self): + project = Project('test', [], None, None) + with self.assertRaises(NoSuchService): + project.get_service('not_found') + + def test_get_service_from_external(self): + web = Service(project='test', name='web') + external_web = Service(project='other', name='web') + external_project = Project('other', [external_web], None, None) + project = Project('test', [web], None, [external_project]) + + self.assertEqual(project.get_service('other_web'), external_web) + def test_get_services_returns_all_services_without_args(self): web = Service( project='figtest', @@ -88,54 +106,58 @@ def test_get_services_returns_all_services_without_args(self): self.assertEqual(project.get_services(), [web, console]) def test_get_services_returns_listed_services_with_args(self): - web = Service( - project='figtest', - name='web', - ) - console = Service( - project='figtest', - name='console', - ) + web = Service(project='figtest', name='web') + console = Service(project='figtest', name='console') project = Project('test', [web, console], None) self.assertEqual(project.get_services(['console']), [console]) def test_get_services_with_include_links(self): - db = Service( - project='figtest', - name='db', - ) + db = Service(project='figtest', name='db') + cache = Service( project='figtest', name='cache') web = Service( project='figtest', name='web', - links=[(db, 'database')] - ) - cache = Service( - project='figtest', - name='cache' + links=[ServiceLink(db, 'database')] ) console = Service( project='figtest', name='console', - links=[(web, 'web')] + links=[ServiceLink(web, 'web')] ) project = Project('test', [web, db, cache, console], None) - self.assertEqual( - project.get_services(['console'], include_links=True), - [db, web, console] - ) + services = project.get_services(['console'], include_links=True) + self.assertEqual(services, [db, web, console]) def test_get_services_removes_duplicates_following_links(self): - db = Service( - project='figtest', - name='db', - ) + db = Service(project='figtest', name='db') web = Service( project='figtest', name='web', - links=[(db, 'database')] + links=[ServiceLink(db, 'database')] ) project = Project('test', [web, db], None) self.assertEqual( project.get_services(['web', 'db'], include_links=True), [db, web] ) + + def test_get_links(self): + db = Service(project='test', name='db') + other = Service(project='test', name='other') + project = Project('test', [db, other], None) + config_links = [ + 'db', + 'db:alias', + 'other', + ] + links = project.get_links(config_links, 'test') + expected = [ + ServiceLink(db, None), + ServiceLink(db, 'alias'), + ServiceLink(other, None), + ] + self.assertEqual(links, expected) + + def test_get_links_no_links(self): + project = Project('test', [], None) + self.assertEqual(project.get_links(None, None), []) From ea3dd3da5771fffed64f87c9ea01f2380e91e116 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 26 Dec 2014 11:12:24 -0500 Subject: [PATCH 2/4] Support includes of external projects. Signed-off-by: Daniel Nephin --- docs/yml.md | 39 +++ fig/includes.py | 157 +++++++++++ fig/project.py | 115 ++++++-- requirements.txt | 1 + setup.py | 1 + .../external-includes-figfile/fig.yml | 20 ++ .../project_b/fig.yml | 23 ++ .../project_c/fig.yml | 14 + tests/integration/cli_test.py | 70 +++++ tests/integration/includes_test.py | 33 +++ tests/unit/includes_test.py | 252 ++++++++++++++++++ tests/unit/project_test.py | 111 ++++++-- 12 files changed, 799 insertions(+), 37 deletions(-) create mode 100644 fig/includes.py create mode 100644 tests/fixtures/external-includes-figfile/fig.yml create mode 100644 tests/fixtures/external-includes-figfile/project_b/fig.yml create mode 100644 tests/fixtures/external-includes-figfile/project_c/fig.yml create mode 100644 tests/integration/includes_test.py create mode 100644 tests/unit/includes_test.py diff --git a/docs/yml.md b/docs/yml.md index a911e450b86..95d599b236f 100644 --- a/docs/yml.md +++ b/docs/yml.md @@ -199,3 +199,42 @@ privileged: true restart: always ``` + +## Project Includes + +External projects can be included by specifying a url to the projects `fig.yml` +file. Only services with `image` may be included (because there would be no way +to build the service without the full project). + +Urls may be filepaths, http/https or s3. Remote files will be cached locally +using the specified cache settings (defaults to a path of ~/.fig-cache/ with +a ttl of 5 minutes). + +Example: + +```yaml + +project-config: + + include: + projecta: + url: 's3://bucket/path/to/key/projecta.yml' + projectb: + url: 'http://example.com/projectb/fig.yml' + projectc: + url: './path/to/projectc/fig.yml' + + # This section is optional, below are the default values + cache: + enable: True + path: ~/.fig-cache/ + ttl: 5min + +webapp: + build: . + links: + - projecta_webapp + - pojrectb_webapp + volumes_from: + - projectc_data +``` diff --git a/fig/includes.py b/fig/includes.py new file mode 100644 index 00000000000..eb8d72ac920 --- /dev/null +++ b/fig/includes.py @@ -0,0 +1,157 @@ +"""Include external projects, allowing services to link to a service +defined in an external project. +""" +import logging +import os +import time + +from pytimeparse import timeparse +import requests +import requests.exceptions +import six +from six.moves.urllib.parse import urlparse, quote +import yaml + +from fig.service import ConfigError + + +log = logging.getLogger(__name__) + + +class FetchExternalConfigError(Exception): + pass + + +def normalize_url(url): + url = urlparse(url) + return url if url.scheme else url._replace(scheme='file') + + +def read_config(content): + return yaml.safe_load(content) + + +def get_project_from_file(url): + # Handle urls in the form file://./some/relative/path + path = url.netloc + url.path if url.netloc.startswith('.') else url.path + with open(path, 'r') as fh: + return read_config(fh.read()) + + +def get_project_from_http(url, config): + try: + response = requests.get( + url.geturl(), + timeout=config.get('timeout', 20), + verify=config.get('verify_ssl_cert', True), + cert=config.get('ssl_cert', None), + proxies=config.get('proxies', None)) + response.raise_for_status() + except requests.exceptions.RequestException as e: + raise FetchExternalConfigError("Failed to include %s: %s" % ( + url.geturl(), e)) + return read_config(response.text) + + +def fetch_external_config(url, include_config): + log.info("Fetching config from %s" % url.geturl()) + + if url.scheme in ('http', 'https'): + return get_project_from_http(url, include_config) + + if url.scheme == 'file': + return get_project_from_file(url) + + raise ConfigError("Unsupported url scheme \"%s\" for %s." % ( + url.scheme, + url)) + + +class LocalConfigCache(object): + + def __init__(self, path, ttl): + self.path = path + self.ttl = ttl + + @classmethod + def from_config(cls, cache_config): + if not cache_config.get('enable', True): + return {} + + path = os.path.expanduser(cache_config.get('path', '~/.fig-cache')) + ttl = timeparse.timeparse(cache_config.get('ttl', '5 min')) + + if not os.path.isdir(path): + os.makedirs(path) + + if ttl is None: + raise ConfigError("Cache ttl \'%s\' could not be parsed" % + cache_config.get('ttl')) + + return cls(path, ttl) + + def is_fresh(self, mtime): + return mtime + self.ttl > time.time() + + def __contains__(self, url): + path = url_to_filename(self.path, url) + return os.path.exists(path) and self.is_fresh(os.path.getmtime(path)) + + def __getitem__(self, url): + if url not in self: + raise KeyError(url) + with open(url_to_filename(self.path, url), 'r') as fh: + return read_config(fh.read()) + + def __setitem__(self, url, contents): + with open(url_to_filename(self.path, url), 'w') as fh: + return fh.write(yaml.dump(contents)) + + +def url_to_filename(path, url): + return os.path.join(path, quote(url.geturl(), safe='')) + + +class ExternalProjectCache(object): + """Cache each Project by the url used to retreive the projects fig.yml. + If multiple projects include the same url, re-use the same instance of the + project. + """ + + def __init__(self, cache, client, factory): + self.config_cache = cache + self.project_cache = {} + self.client = client + self.factory = factory + + def get_project_from_include(self, name, include): + if 'url' not in include: + raise ConfigError("Project include '%s' requires a url" % name) + url = normalize_url(include['url']) + + if url not in self.project_cache: + config = self.get_config(url, include) + self.project_cache[url] = self.build_project(name, config) + + return self.project_cache[url] + + def get_config(self, url, include): + if url in self.config_cache: + return self.config_cache[url] + + self.config_cache[url] = config = fetch_external_config(url, include) + return config + + def build_project(self, name, config): + def is_build_service(service_name, service): + if 'build' not in service: + return False + log.info("Service %s_%s is external and uses build, skipping" % ( + name, + service_name)) + return True + + config = dict( + (name, service) for name, service in six.iteritems(config) + if not is_build_service(name, service)) + return self.factory(name, config, self.client, project_cache=self) diff --git a/fig/project.py b/fig/project.py index 3634fcdd251..db97fa88194 100644 --- a/fig/project.py +++ b/fig/project.py @@ -1,17 +1,28 @@ from __future__ import unicode_literals from __future__ import absolute_import +from itertools import chain import logging +from operator import ( + attrgetter, + itemgetter, +) -from .service import Service -from .container import Container from docker.errors import APIError +import six + +from fig import includes +from fig.service import ( + Service, + ServiceLink, +) +from fig.container import Container log = logging.getLogger(__name__) def sort_service_dicts(services): # Topological sort (Cormen/Tarjan algorithm). - unmarked = services[:] + unmarked = sorted(services, key=itemgetter('name')) temporary_marked = set() sorted_services = [] @@ -44,45 +55,85 @@ class Project(object): """ A collection of services. """ - def __init__(self, name, services, client): + def __init__(self, name, services, client, namespace=None, external_projects=None): self.name = name self.services = services self.client = client + # The top level project name is the namespace for included projects + self.namespace = namespace or name + self.external_projects = external_projects or [] @classmethod - def from_dicts(cls, name, service_dicts, client): + def from_dicts(cls, name, service_dicts, client, namespace, external_projects): """ Construct a ServiceCollection from a list of dicts representing services. """ - project = cls(name, [], client) + project = cls(name, [], client, namespace, external_projects) for service_dict in sort_service_dicts(service_dicts): - links = project.get_links(service_dict) + links = project.get_links(service_dict.pop('links', None), + service_dict['name']) volumes_from = project.get_volumes_from(service_dict) - project.services.append(Service(client=client, project=name, links=links, volumes_from=volumes_from, **service_dict)) + project.services.append( + Service(client=client, + project=name, + links=links, + volumes_from=volumes_from, + **service_dict)) return project @classmethod - def from_config(cls, name, config, client): - dicts = [] + def from_config(cls, name, config, client, namespace=None, project_cache=None): + services = [] + project_config = config.pop('project-config', {}) + external_projects = get_external_projects( + project_config.pop('include', {}), + project_config.pop('cache', {}), + client, + name, + project_cache) + for service_name, service in list(config.items()): if not isinstance(service, dict): - raise ConfigurationError('Service "%s" doesn\'t have any configuration options. All top level keys in your fig.yml must map to a dictionary of configuration options.' % service_name) + raise ConfigurationError( + 'Service "%s" doesn\'t have any configuration options. ' + 'All top level keys in your fig.yml must map to a ' + 'dictionary of configuration options.' % service_name) service['name'] = service_name - dicts.append(service) - return cls.from_dicts(name, dicts, client) + services.append(service) + return cls.from_dicts(name, services, client, namespace, external_projects) def get_service(self, name): + """Retrieve a service by name. + + :param name: name of the service + :returns: :class:`fig.service.Service` + :raises NoSuchService: if no service was found by that name """ - Retrieve a service by name. Raises NoSuchService - if the named service does not exist. - """ - for service in self.services: - if service.name == name: - return service + if '_' in name: + project_name, service_name = name.rsplit('_', 1) + if project_name != self.namespace: + # References (link, etc) do not contain the namespace, so add it + project_name = self.namespace + project_name + else: + project_name, service_name = self.name, name + + if project_name == self.name: + for service in self.services: + if service.name == service_name: + return service + + for project in self.external_projects: + if project.name == project_name: + return project.get_service(service_name) raise NoSuchService(name) + @property + def all_services(self): + return (flat_map(attrgetter('services'), self.external_projects) + + self.services) + def get_services(self, service_names=None, include_links=False): """ Returns a list of this project's services filtered @@ -222,6 +273,32 @@ def flat_map(func, seq): return list(chain.from_iterable(map(func, seq))) +def get_external_projects( + includes_config, + cache_config, + client, + project_name, + project_cache): + """Recursively fetch included projects. + + Cache each external project by url. If a project is encountered with the + same url the same instance of :class:`Project` will be returned. + """ + def build_project(name, *args, **kwargs): + name = '%s%s' % (project_name, name) + kwargs['namespace'] = project_name + return Project.from_config(name, *args, **kwargs) + + if project_cache is None: + project_cache = includes.ExternalProjectCache( + includes.LocalConfigCache.from_config(cache_config), + client, + build_project) + + return [project_cache.get_project_from_include(*item) + for item in six.iteritems(includes_config)] + + class NoSuchService(Exception): def __init__(self, name): self.name = name diff --git a/requirements.txt b/requirements.txt index 2ccdf59a258..0b17acdd8bb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,7 @@ PyYAML==3.10 docker-py==0.6.0 dockerpty==0.3.2 docopt==0.6.1 +pytimeparse==1.1.2 requests==2.2.1 six==1.7.3 texttable==0.8.1 diff --git a/setup.py b/setup.py index 4cf8e589d98..aba151ad902 100644 --- a/setup.py +++ b/setup.py @@ -33,6 +33,7 @@ def find_version(*file_paths): 'docker-py >= 0.6.0, < 0.7', 'dockerpty >= 0.3.2, < 0.4', 'six >= 1.3.0, < 2', + 'pytimeparse >= 1.1.2', ] tests_require = [ diff --git a/tests/fixtures/external-includes-figfile/fig.yml b/tests/fixtures/external-includes-figfile/fig.yml new file mode 100644 index 00000000000..ad10475872f --- /dev/null +++ b/tests/fixtures/external-includes-figfile/fig.yml @@ -0,0 +1,20 @@ + +project-config: + include: + projectb: + url: ./tests/fixtures/external-includes-figfile/project_b/fig.yml + + projectc: + url: ./tests/fixtures/external-includes-figfile/project_c/fig.yml + +db: + image: busybox:latest + command: /bin/sleep 300 + +webapp: + image: busybox:latest + command: /bin/sleep 300 + links: + - db + - projectb_webapp + - projectc_webapp diff --git a/tests/fixtures/external-includes-figfile/project_b/fig.yml b/tests/fixtures/external-includes-figfile/project_b/fig.yml new file mode 100644 index 00000000000..5d45dc4a1ef --- /dev/null +++ b/tests/fixtures/external-includes-figfile/project_b/fig.yml @@ -0,0 +1,23 @@ + +project-config: + include: + proejctc: + url: ./tests/fixtures/external-includes-figfile/project_c/fig.yml + +db: + image: busybox:latest + command: /bin/sleep 300 + +configs: + image: busybox:latest + volumes: + - /home + +webapp: + image: busybox:latest + command: /bin/sleep 300 + links: + - db + - projectc_webapp + volumes_from: + - configs diff --git a/tests/fixtures/external-includes-figfile/project_c/fig.yml b/tests/fixtures/external-includes-figfile/project_c/fig.yml new file mode 100644 index 00000000000..9d5e595ad87 --- /dev/null +++ b/tests/fixtures/external-includes-figfile/project_c/fig.yml @@ -0,0 +1,14 @@ + +configs: + image: busybox:latest + volumes: + - /home + +webapp: + image: busybox:latest + command: /bin/sleep 300 + volumes_from: + - configs + +unrelated: + image: busybox:latest diff --git a/tests/integration/cli_test.py b/tests/integration/cli_test.py index 21e025708cb..be849550df3 100644 --- a/tests/integration/cli_test.py +++ b/tests/integration/cli_test.py @@ -120,6 +120,60 @@ def test_up_with_no_deps(self): self.assertEqual(len(db.containers()), 0) self.assertEqual(len(console.containers()), 0) + def test_up_with_includes(self): + self.command.base_dir = 'tests/fixtures/external-includes-figfile' + + self.assertEqual( + [s.full_name for s in self.project.get_services()], + [ + 'externalincludesfigfileprojectc_configs', + 'externalincludesfigfileprojectc_unrelated', + 'externalincludesfigfileprojectc_webapp', + 'externalincludesfigfileprojectb_configs', + 'externalincludesfigfileprojectb_db', + 'externalincludesfigfileprojectb_webapp', + 'externalincludesfigfile_db', + 'externalincludesfigfile_webapp', + ]) + + self.command.dispatch(['up', '-d'], None) + + self.assertEqual( + set(c.name for c in self.project.containers(stopped=True)), + set([ + 'externalincludesfigfile_db_1', + 'externalincludesfigfile_webapp_1', + 'externalincludesfigfileprojectb_configs_1', + 'externalincludesfigfileprojectb_db_1', + 'externalincludesfigfileprojectb_webapp_1', + 'externalincludesfigfileprojectc_configs_1', + 'externalincludesfigfileprojectc_unrelated_1', + 'externalincludesfigfileprojectc_webapp_1', + ])) + + assert_has_links_and_volumes( + self.project, + 'webapp', + set([ + 'externalincludesfigfile_db_1', + 'externalincludesfigfileprojectb_webapp_1', + 'externalincludesfigfileprojectc_webapp_1', + ])) + + assert_has_links_and_volumes( + self.project, + 'projectb_webapp', + set([ + 'externalincludesfigfileprojectb_db_1', + 'externalincludesfigfileprojectc_webapp_1' + ]), + ['/home']) + + assert_has_links_and_volumes( + self.project, + 'projectc_webapp', + expected_volumes=['/home']) + def test_up_with_recreate(self): self.command.dispatch(['up', '-d'], None) service = self.project.get_service('simple') @@ -382,3 +436,19 @@ def get_port(number, mock_stdout): self.assertEqual(get_port(3000), container.get_local_port(3000)) self.assertEqual(get_port(3001), "0.0.0.0:9999") self.assertEqual(get_port(3002), "") + + +def assert_has_links_and_volumes( + project, + service_name, + expected_links=None, + expected_volumes=None): + + container = project.get_service(service_name).get_container() + + if expected_links is not None: + assert expected_links.issubset(set(container.links())) + + # TODO: use container.get() + if expected_volumes is not None: + assert expected_volumes == container.inspect()['VolumesRW'].keys() diff --git a/tests/integration/includes_test.py b/tests/integration/includes_test.py new file mode 100644 index 00000000000..81e13b7ef19 --- /dev/null +++ b/tests/integration/includes_test.py @@ -0,0 +1,33 @@ + +from .. import unittest + +from fig.includes import ( + FetchExternalConfigError, + get_project_from_http, + get_project_from_s3, + normalize_url, +) + + +class IncludeHttpTest(unittest.TestCase): + + def test_get_project_from_http(self): + # returns JSON, but yaml can parse that just fine + url = normalize_url('http://httpbin.org/get') + project = get_project_from_http(url, {}) + self.assertIn('url', project) + + def test_get_project_from_http_with_http_error(self): + url = normalize_url('http://httpbin.org/status/404') + with self.assertRaises(FetchExternalConfigError) as exc_context: + get_project_from_http(url, {}) + self.assertEqual( + 'Failed to include http://httpbin.org/status/404: ' + '404 Client Error: NOT FOUND', + str(exc_context.exception)) + + def test_get_project_from_http_with_connection_error(self): + url = normalize_url('http://hostdoesnotexist.bogus/') + with self.assertRaises(FetchExternalConfigError) as exc_context: + get_project_from_http(url, {'timeout': 2}) + self.assertIn('Name or service not known', str(exc_context.exception)) diff --git a/tests/unit/includes_test.py b/tests/unit/includes_test.py new file mode 100644 index 00000000000..30728cc0cb8 --- /dev/null +++ b/tests/unit/includes_test.py @@ -0,0 +1,252 @@ +import contextlib +import os.path +import shutil +import tempfile + +import boto.exception +import boto.s3.connection +import mock +from .. import unittest + +from fig.includes import ( + ExternalProjectCache, + FetchExternalConfigError, + LocalConfigCache, + fetch_external_config, + get_project_from_file, + get_project_from_s3, + normalize_url, + url_to_filename, +) +from fig.project import Project +from fig.service import ConfigError + + +class NormalizeUrlTest(unittest.TestCase): + + def test_normalize_url_with_scheme(self): + url = normalize_url('HTTPS://example.com') + self.assertEqual(url.scheme, 'https') + + def test_normalize_url_without_scheme(self): + url = normalize_url('./path/to/somewhere') + self.assertEqual(url.scheme, 'file') + + +class GetProjectFromS3Test(unittest.TestCase): + + @mock.patch('fig.includes.get_boto_conn', autospec=True) + def test_get_project_from_s3(self, mock_get_conn): + mock_bucket = mock_get_conn.return_value.get_bucket.return_value + mock_key = mock_bucket.get_key.return_value + mock_key.get_contents_as_string.return_value = 'foo:\n build: .' + url = normalize_url('s3://bucket/path/to/key/fig.yml') + + project = get_project_from_s3(url) + self.assertEqual(project, {'foo': {'build': '.'}}) + + mock_get_conn.assert_called_once_with() + mock_get_conn.return_value.get_bucket.assert_called_once_with('bucket') + mock_bucket.get_key.assert_called_once_with('/path/to/key/fig.yml') + + + @mock.patch('fig.includes.get_boto_conn', autospec=True) + def test_get_project_from_s3_not_found(self, mock_get_conn): + mock_bucket = mock_get_conn.return_value.get_bucket.return_value + mock_bucket.get_key.return_value = None + url = normalize_url('s3://bucket/path/to/key/fig.yml') + + with self.assertRaises(FetchExternalConfigError) as exc_context: + get_project_from_s3(url) + self.assertEqual( + "Failed to include %s: Not Found" % url.geturl(), + str(exc_context.exception)) + + @mock.patch('fig.includes.get_boto_conn', autospec=True) + def test_get_project_from_s3_bucket_error(self, mock_get_conn): + mock_get_bucket = mock_get_conn.return_value.get_bucket + mock_get_bucket.side_effect = boto.exception.S3ResponseError( + 404, "Bucket Not Found") + + url = normalize_url('s3://bucket/path/to/key/fig.yml') + with self.assertRaises(FetchExternalConfigError) as exc_context: + get_project_from_s3(url) + self.assertEqual( + "Failed to include %s: S3ResponseError: 404 Bucket Not Found\n" % + url.geturl(), str(exc_context.exception)) + + +class FetchExternalConfigTest(unittest.TestCase): + + def test_unsupported_scheme(self): + with self.assertRaises(ConfigError) as exc: + fetch_external_config(normalize_url("bogus://something"), None) + self.assertIn("bogus", str(exc.exception)) + + def test_fetch_from_file(self): + url = "./tests/fixtures/external-includes-figfile/fig.yml" + config = fetch_external_config(normalize_url(url), None) + self.assertEqual( + set(config.keys()), + set(['db', 'webapp', 'project-config'])) + + +class GetProjectFromFileWithNormalizeUrlTest(unittest.TestCase): + + def setUp(self): + self.expected = set(['db', 'webapp', 'project-config']) + self.path = "tests/fixtures/external-includes-figfile/fig.yml" + + def test_fetch_from_file_relative_no_context(self): + config = get_project_from_file(normalize_url(self.path)) + self.assertEqual(set(config.keys()), self.expected) + + def test_fetch_from_file_relative_with_context(self): + url = './' + self.path + config = get_project_from_file(normalize_url(url)) + self.assertEqual(set(config.keys()), self.expected) + + def test_fetch_from_file_absolute_path(self): + url = os.path.abspath(self.path) + config = get_project_from_file(normalize_url(url)) + self.assertEqual(set(config.keys()), self.expected) + + def test_fetch_from_file_relative_with_scheme(self): + url = 'file://./' + self.path + config = get_project_from_file(normalize_url(url)) + self.assertEqual(set(config.keys()), self.expected) + + def test_fetch_from_file_absolute_with_scheme(self): + url = 'file://' + os.path.abspath(self.path) + config = get_project_from_file(normalize_url(url)) + self.assertEqual(set(config.keys()), self.expected) + + +class LocalConfigCacheTest(unittest.TestCase): + + url = "http://example.com/path/to/file.yml" + + def setUp(self): + self.path = '/base/path' + self.ttl = 20 + + def test_url_to_filename(self): + path = '/base' + filename = url_to_filename(path, normalize_url(self.url)) + expected = '/base/http%3A%2F%2Fexample.com%2Fpath%2Fto%2Ffile.yml' + self.assertEqual(filename, expected) + + @mock.patch('fig.includes.time.time', autospec=True) + def test_is_fresh_false(self, mock_time): + cache = LocalConfigCache(self.path, self.ttl) + mock_time.return_value = 1000 + self.assertFalse(cache.is_fresh(900)) + + @mock.patch('fig.includes.time.time', autospec=True) + def test_is_fresh_true(self, mock_time): + cache = LocalConfigCache(self.path, self.ttl) + mock_time.return_value = 1000 + self.assertTrue(cache.is_fresh(990)) + + @mock.patch('fig.includes.os.path.isdir', autospec=True) + def test_from_config(self, mock_isdir): + mock_isdir.return_value = True + cache = LocalConfigCache.from_config({ + 'ttl': '6 min', + 'path': '~/.cache-path' + }) + self.assertEqual(cache.ttl, 360) + self.assertEqual(cache.path, os.path.expandvars('$HOME/.cache-path')) + + def test_get_and_set(self): + url = normalize_url(self.url) + config = {'foo': {'image': 'busybox'}} + + with temp_dir() as path: + cache = LocalConfigCache(path, self.ttl) + + self.assertNotIn(url, cache) + cache[url] = config + self.assertIn(url, cache) + self.assertEqual(config, cache[url]) + + +@contextlib.contextmanager +def temp_dir(): + path = tempfile.mkdtemp() + try: + yield path + finally: + shutil.rmtree(path) + + +class ExternalProjectCacheTest(unittest.TestCase): + + project_b = { + 'url': 'http://example.com/project_b/fig.yml' + } + + def setUp(self): + self.client = mock.Mock() + self.mock_factory = mock.create_autospec(Project.from_config) + self.externals = ExternalProjectCache({}, self.client, self.mock_factory) + + def test_get_project_from_include_invalid_config(self): + with self.assertRaises(ConfigError) as exc: + self.externals.get_project_from_include('something', {}) + self.assertIn( + "Project include 'something' requires a url", + str(exc.exception)) + + @mock.patch('fig.includes.fetch_external_config', autospec=True) + def test_get_external_projects_no_cache(self, mock_fetch): + name = 'project_b' + project = self.externals.get_project_from_include(name, self.project_b) + + url = normalize_url(self.project_b['url']) + mock_fetch.assert_called_once_with(url, self.project_b) + + self.mock_factory.assert_called_once_with( + name, + {}, + self.client, + project_cache=self.externals) + + self.assertEqual(project, self.mock_factory.return_value) + self.assertIn(url, self.externals.config_cache) + self.assertIn(url, self.externals.project_cache) + + @mock.patch('fig.includes.fetch_external_config', autospec=True) + def test_get_external_projects_from_cache(self, mock_fetch): + mock_project = mock.create_autospec(Project) + url = normalize_url(self.project_b['url']) + self.externals.project_cache[url] = mock_project + + name = 'project_b' + project = self.externals.get_project_from_include(name, self.project_b) + + self.assertEqual(mock_fetch.called, False) + self.assertEqual(self.mock_factory.called, False) + self.assertEqual(project, mock_project) + + def test_build_project_with_build_directive(self): + config = { + 'foo': {'build': '.'}, + 'other': {'image': 'busybox'}, + } + self.externals.factory = Project.from_config + with mock.patch('fig.includes.log', autospec=True) as mock_log: + project = self.externals.build_project('project', config) + self.assertEqual(len(project.services), 1) + self.assertEqual(project.services[0].full_name, 'project_other') + + mock_log.info.assert_called_once_with( + "Service project_foo is external and uses build, skipping") + + def test_get_config_from_config_cache(self): + url = normalize_url(self.project_b['url']) + mock_config = mock.Mock() + self.externals.config_cache = {url: mock_config} + + config = self.externals.get_config(url, self.project_b) + self.assertEqual(mock_config, config) diff --git a/tests/unit/project_test.py b/tests/unit/project_test.py index d6c6c44677a..274bd33345a 100644 --- a/tests/unit/project_test.py +++ b/tests/unit/project_test.py @@ -1,7 +1,21 @@ from __future__ import unicode_literals + +import docker +import mock from .. import unittest -from fig.service import Service -from fig.project import Project, ConfigurationError + +from fig import includes +from fig.service import ( + Service, + ServiceLink, +) +from fig.project import ( + ConfigurationError, + NoSuchService, + Project, + get_external_projects, +) + class ProjectTest(unittest.TestCase): def test_from_dict(self): @@ -14,7 +28,7 @@ def test_from_dict(self): 'name': 'db', 'image': 'busybox:latest' }, - ], None) + ], None, None, None) self.assertEqual(len(project.services), 2) self.assertEqual(project.get_service('web').name, 'web') self.assertEqual(project.get_service('web').options['image'], 'busybox:latest') @@ -38,7 +52,7 @@ def test_from_dict_sorts_in_dependency_order(self): 'image': 'busybox:latest', 'volumes': ['/tmp'], } - ], None) + ], None, None, None) self.assertEqual(project.services[0].name, 'volume') self.assertEqual(project.services[1].name, 'db') @@ -72,11 +86,11 @@ def test_get_service_no_external(self): client=None, image="busybox:latest", ) - project = Project('test', [web], None) + project = Project('test', [web], None, None) self.assertEqual(project.get_service('web'), web) def test_get_service_with_project_name(self): - web = Service( project='figtest', name='web') + web = Service(project='figtest', name='web') project = Project('test', [web], None, None) self.assertEqual(project.get_service('test_web'), web) @@ -86,24 +100,26 @@ def test_get_service_not_found(self): project.get_service('not_found') def test_get_service_from_external(self): + project_name = 'theproject' web = Service(project='test', name='web') external_web = Service(project='other', name='web') - external_project = Project('other', [external_web], None, None) - project = Project('test', [web], None, [external_project]) + external_project = Project( + project_name + 'other', + [external_web], + None, + project_name) + project = Project(project_name, [web], None, None, [external_project]) self.assertEqual(project.get_service('other_web'), external_web) def test_get_services_returns_all_services_without_args(self): - web = Service( - project='figtest', - name='web', - ) - console = Service( - project='figtest', - name='console', - ) - project = Project('test', [web, console], None) - self.assertEqual(project.get_services(), [web, console]) + web = Service(project='figtest', name='web') + console = Service(project='figtest', name='console') + external_web = Service(project='servicea', name='web') + + external_projects = [Project('servicea',[external_web], None, None)] + project = Project('test', [web, console], None, None, external_projects) + self.assertEqual(project.get_services(), [external_web, web, console]) def test_get_services_returns_listed_services_with_args(self): web = Service(project='figtest', name='web') @@ -161,3 +177,62 @@ def test_get_links(self): def test_get_links_no_links(self): project = Project('test', [], None) self.assertEqual(project.get_links(None, None), []) + + +class GetExternalProjectsTest(unittest.TestCase): + + project_includes = { + 'project_b': { + 'url': 'http://example.com/project_b/fig.yml' + }, + 'project_c': { + 'url': 'http://example.com/project_c/fig.yml' + }, + } + + def setUp(self): + self.mock_client = mock.create_autospec(docker.Client) + self.mock_externals_cache = mock.create_autospec( + includes.ExternalProjectCache) + self.project_name = 'rootproject' + + def test_get_external_projects_none(self): + self.assertEqual(get_external_projects({}, {}, None, None, None), []) + + @mock.patch('fig.project.includes.LocalConfigCache', autospec=True) + @mock.patch('fig.project.includes.ExternalProjectCache', autospec=True) + def test_get_external_projects_empty_cache( + self, + mock_externals_cache, + mock_config_cache): + projects = get_external_projects( + self.project_includes, + {'ttl': 30}, + self.mock_client, + self.project_name, + None) + + self.assertEqual(len(projects), 2) + mock_externals_cache.assert_called_once_with( + mock_config_cache.from_config.return_value, + self.mock_client, + mock.ANY) + + mock_config_cache.from_config.assert_called_once_with({'ttl': 30}) + + _, _, build_project = mock_externals_cache.mock_calls[0][1] + project = build_project('included', {}, self.mock_client) + self.assertEqual(project.name, 'rootprojectincluded') + + def test_get_external_projects_with_cache(self): + projects = get_external_projects( + self.project_includes, + {}, + self.mock_client, + self.project_name, + self.mock_externals_cache) + + self.assertEqual( + [self.mock_externals_cache.get_project_from_include.return_value + for _ in range(2)], + projects) From 5b82cdd073b398224eca45ecbd9ec4393713d389 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 16 Dec 2014 17:44:17 -0800 Subject: [PATCH 3/4] Add support for s3 paths. Signed-off-by: Daniel Nephin --- fig/cli/main.py | 4 ++-- fig/includes.py | 27 +++++++++++++++++++++++++++ requirements-dev.txt | 1 + tests/integration/cli_test.py | 3 +-- tests/unit/cli_test.py | 3 ++- 5 files changed, 33 insertions(+), 5 deletions(-) diff --git a/fig/cli/main.py b/fig/cli/main.py index 8cdaee62090..3c55acb8c4a 100644 --- a/fig/cli/main.py +++ b/fig/cli/main.py @@ -56,8 +56,8 @@ def setup_logging(): root_logger.addHandler(console_handler) root_logger.setLevel(logging.DEBUG) - # Disable requests logging - logging.getLogger("requests").propagate = False + logging.getLogger("requests").setLevel(logging.WARN) + logging.getLogger("boto").setLevel(logging.WARN) # stolen from docopt master diff --git a/fig/includes.py b/fig/includes.py index eb8d72ac920..240d670cb97 100644 --- a/fig/includes.py +++ b/fig/includes.py @@ -53,6 +53,30 @@ def get_project_from_http(url, config): return read_config(response.text) +# Return the connection from a function, so it can be mocked in tests +def get_boto_conn(): + # Local import so that boto is only a dependency if it's used + import boto.s3.connection + return boto.s3.connection.S3Connection() + + +def get_project_from_s3(url): + import boto.exception + try: + conn = get_boto_conn() + bucket = conn.get_bucket(url.netloc) + except (boto.exception.BotoServerError, boto.exception.BotoClientError) as e: + raise FetchExternalConfigError( + "Failed to include %s: %s" % (url.geturl(), e)) + + key = bucket.get_key(url.path) + if not key: + raise FetchExternalConfigError( + "Failed to include %s: Not Found" % url.geturl()) + + return read_config(key.get_contents_as_string()) + + def fetch_external_config(url, include_config): log.info("Fetching config from %s" % url.geturl()) @@ -62,6 +86,9 @@ def fetch_external_config(url, include_config): if url.scheme == 'file': return get_project_from_file(url) + if url.scheme == 's3': + return get_project_from_s3(url) + raise ConfigError("Unsupported url scheme \"%s\" for %s." % ( url.scheme, url)) diff --git a/requirements-dev.txt b/requirements-dev.txt index bd5a3949387..4c6be38a585 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -3,3 +3,4 @@ nose git+https://github.com/pyinstaller/pyinstaller.git@12e40471c77f588ea5be352f7219c873ddaae056#egg=pyinstaller unittest2 flake8 +boto diff --git a/tests/integration/cli_test.py b/tests/integration/cli_test.py index be849550df3..a535f62d893 100644 --- a/tests/integration/cli_test.py +++ b/tests/integration/cli_test.py @@ -449,6 +449,5 @@ def assert_has_links_and_volumes( if expected_links is not None: assert expected_links.issubset(set(container.links())) - # TODO: use container.get() if expected_volumes is not None: - assert expected_volumes == container.inspect()['VolumesRW'].keys() + assert expected_volumes == container.get('Volumes').keys() diff --git a/tests/unit/cli_test.py b/tests/unit/cli_test.py index c9151165ee0..5928373447d 100644 --- a/tests/unit/cli_test.py +++ b/tests/unit/cli_test.py @@ -66,4 +66,5 @@ def test_help(self): def test_setup_logging(self): main.setup_logging() self.assertEqual(logging.getLogger().level, logging.DEBUG) - self.assertEqual(logging.getLogger('requests').propagate, False) + self.assertEqual(logging.getLogger('requests').level, logging.WARN) + self.assertEqual(logging.getLogger('boto').level, logging.WARN) From b31f9a21b88f0ca578c30a65e4e2297dfa120176 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 6 Feb 2015 15:36:51 -0800 Subject: [PATCH 4/4] Fix a race condition while creating the fig cache directory. Signed-off-by: Daniel Nephin --- fig/includes.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/fig/includes.py b/fig/includes.py index 240d670cb97..f6fa2784fa2 100644 --- a/fig/includes.py +++ b/fig/includes.py @@ -109,7 +109,13 @@ def from_config(cls, cache_config): ttl = timeparse.timeparse(cache_config.get('ttl', '5 min')) if not os.path.isdir(path): - os.makedirs(path) + try: + os.makedirs(path) + except OSError: + # Handle the race condition where some other process creates + # this directory after the isdir check + if not os.path.isdir(path): + raise if ttl is None: raise ConfigError("Cache ttl \'%s\' could not be parsed" %