diff --git a/compose/cli/command.py b/compose/cli/command.py index 1a9bc3dcbf3..525217ee75a 100644 --- a/compose/cli/command.py +++ b/compose/cli/command.py @@ -49,7 +49,10 @@ def project_from_options(base_dir, options): base_dir, get_config_path(options.get('--file')), project_name=options.get('--project-name'), - verbose=options.get('--verbose')) + verbose=options.get('--verbose'), + use_networking=options.get('--x-networking'), + network_driver=options.get('--x-network-driver'), + ) def get_config_path(file_option): @@ -64,8 +67,8 @@ def get_config_path(file_option): return [config_file] if config_file else None -def get_client(verbose=False): - client = docker_client() +def get_client(verbose=False, version=None): + client = docker_client(version=version) if verbose: version_info = six.iteritems(client.version()) log.info("Compose version %s", __version__) @@ -76,14 +79,19 @@ def get_client(verbose=False): return client -def get_project(base_dir, config_path=None, project_name=None, verbose=False): +def get_project(base_dir, config_path=None, project_name=None, verbose=False, + use_networking=False, network_driver=None): config_details = config.find(base_dir, config_path) + api_version = '1.21' if use_networking else None try: return Project.from_dicts( get_project_name(config_details.working_dir, project_name), config.load(config_details), - get_client(verbose=verbose)) + get_client(verbose=verbose, version=api_version), + use_networking=use_networking, + network_driver=network_driver, + ) except ConfigError as e: raise errors.UserError(six.text_type(e)) diff --git a/compose/cli/docker_client.py b/compose/cli/docker_client.py index 2c634f33767..734f4237b04 100644 --- a/compose/cli/docker_client.py +++ b/compose/cli/docker_client.py @@ -9,7 +9,10 @@ log = logging.getLogger(__name__) -def docker_client(): +DEFAULT_API_VERSION = '1.19' + + +def docker_client(version=None): """ Returns a docker-py client configured using environment variables according to the same logic as the official Docker client. @@ -18,6 +21,8 @@ def docker_client(): log.warn('The DOCKER_CLIENT_TIMEOUT environment variable is deprecated. Please use COMPOSE_HTTP_TIMEOUT instead.') kwargs = kwargs_from_env(assert_hostname=False) - kwargs['version'] = os.environ.get('COMPOSE_API_VERSION', '1.19') + kwargs['version'] = version or os.environ.get( + 'COMPOSE_API_VERSION', + DEFAULT_API_VERSION) kwargs['timeout'] = HTTP_TIMEOUT return Client(**kwargs) diff --git a/compose/cli/main.py b/compose/cli/main.py index 0f0a69cad63..c800d95f98c 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -117,6 +117,10 @@ class TopLevelCommand(DocoptCommand): Options: -f, --file FILE Specify an alternate compose file (default: docker-compose.yml) -p, --project-name NAME Specify an alternate project name (default: directory name) + --x-networking (EXPERIMENTAL) Use new Docker networking functionality. + Requires Docker 1.9 or later. + --x-network-driver DRIVER (EXPERIMENTAL) Specify a network driver (default: "bridge"). + Requires Docker 1.9 or later. --verbose Show more output -v, --version Print version and exit diff --git a/compose/project.py b/compose/project.py index 999c2890411..0e20a4cee74 100644 --- a/compose/project.py +++ b/compose/project.py @@ -77,10 +77,12 @@ class Project(object): """ A collection of services. """ - def __init__(self, name, services, client): + def __init__(self, name, services, client, use_networking=False, network_driver=None): self.name = name self.services = services self.client = client + self.use_networking = use_networking + self.network_driver = network_driver or 'bridge' def labels(self, one_off=False): return [ @@ -89,11 +91,15 @@ def labels(self, one_off=False): ] @classmethod - def from_dicts(cls, name, service_dicts, client): + def from_dicts(cls, name, service_dicts, client, use_networking=False, network_driver=None): """ Construct a ServiceCollection from a list of dicts representing services. """ - project = cls(name, [], client) + project = cls(name, [], client, use_networking=use_networking, network_driver=network_driver) + + if use_networking: + remove_links(service_dicts) + for service_dict in sort_service_dicts(service_dicts): links = project.get_links(service_dict) volumes_from = project.get_volumes_from(service_dict) @@ -103,6 +109,7 @@ def from_dicts(cls, name, service_dicts, client): Service( client=client, project=name, + use_networking=use_networking, links=links, net=net, volumes_from=volumes_from, @@ -207,6 +214,8 @@ def get_volumes_from(self, service_dict): def get_net(self, service_dict): net = service_dict.pop('net', None) if not net: + if self.use_networking: + return Net(self.name) return Net(None) net_name = get_service_name_from_net(net) @@ -289,6 +298,9 @@ def up(self, plans = self._get_convergence_plans(services, strategy) + if self.use_networking: + self.ensure_network_exists() + return [ container for service in services @@ -350,6 +362,26 @@ def matches_service_names(container): return [c for c in containers if matches_service_names(c)] + def get_network(self): + networks = self.client.networks(names=[self.name]) + if networks: + return networks[0] + return None + + def ensure_network_exists(self): + # TODO: recreate network if driver has changed? + if self.get_network() is None: + log.info( + 'Creating network "{}" with driver "{}"' + .format(self.name, self.network_driver) + ) + self.client.create_network(self.name, driver=self.network_driver) + + def remove_network(self): + network = self.get_network() + if network: + self.client.remove_network(network['id']) + def _inject_deps(self, acc, service): dep_names = service.get_dependency_names() @@ -365,6 +397,26 @@ def _inject_deps(self, acc, service): return acc + dep_services +def remove_links(service_dicts): + services_with_links = [s for s in service_dicts if 'links' in s] + if not services_with_links: + return + + if len(services_with_links) == 1: + prefix = '"{}" defines'.format(services_with_links[0]['name']) + else: + prefix = 'Some services ({}) define'.format( + ", ".join('"{}"'.format(s['name']) for s in services_with_links)) + + log.warn( + '\n{} links, which are not compatible with Docker networking and will be ignored.\n' + 'Future versions of Docker will not support links - you should remove them for ' + 'forwards-compatibility.\n'.format(prefix)) + + for s in services_with_links: + del s['links'] + + class NoSuchService(Exception): def __init__(self, name): self.name = name diff --git a/compose/service.py b/compose/service.py index 0d89afc02d0..5f1d59468c1 100644 --- a/compose/service.py +++ b/compose/service.py @@ -113,6 +113,7 @@ def __init__( name, client=None, project='default', + use_networking=False, links=None, volumes_from=None, net=None, @@ -124,6 +125,7 @@ def __init__( self.name = name self.client = client self.project = project + self.use_networking = use_networking self.links = links or [] self.volumes_from = volumes_from or [] self.net = net or Net(None) @@ -602,6 +604,9 @@ def _get_container_create_options( container_options['hostname'] = parts[0] container_options['domainname'] = parts[2] + if 'hostname' not in container_options and self.use_networking: + container_options['hostname'] = self.name + if 'ports' in container_options or 'expose' in self.options: ports = [] all_ports = container_options.get('ports', []) + self.options.get('expose', []) diff --git a/docs/django.md b/docs/django.md index c7ebf58bfe4..2ebf4b4b97d 100644 --- a/docs/django.md +++ b/docs/django.md @@ -64,9 +64,8 @@ and a `docker-compose.yml` file. The `docker-compose.yml` file describes the services that make your app. In this example those services are a web server and database. The compose file - also describes which Docker images these services use, how they link - together, any volumes they might need mounted inside the containers. - Finally, the `docker-compose.yml` file describes which ports these services + also describes which Docker images these services use, any volumes they might + need mounted inside the containers, and any ports they might expose. See the [`docker-compose.yml` reference](yml.md) for more information on how this file works. @@ -81,8 +80,6 @@ and a `docker-compose.yml` file. - .:/code ports: - "8000:8000" - links: - - db This file defines two services: The `db` service and the `web` service. diff --git a/docs/index.md b/docs/index.md index 2e10e080157..e19e7d7f445 100644 --- a/docs/index.md +++ b/docs/index.md @@ -128,8 +128,6 @@ Next, define a set of services using `docker-compose.yml`: - "5000:5000" volumes: - .:/code - links: - - redis redis: image: redis @@ -138,7 +136,6 @@ This template defines two services, `web` and `redis`. The `web` service: * Builds from the `Dockerfile` in the current directory. * Forwards the exposed port 5000 on the container to port 5000 on the host machine. * Mounts the current directory on the host to `/code` inside the container allowing you to modify the code without having to rebuild the image. -* Links the web container to the Redis service. The `redis` service uses the latest public [Redis](https://registry.hub.docker.com/_/redis/) image pulled from the Docker Hub registry. diff --git a/docs/networking.md b/docs/networking.md new file mode 100644 index 00000000000..f4227917acf --- /dev/null +++ b/docs/networking.md @@ -0,0 +1,84 @@ + + + +# Networking in Compose + +> **Note:** Compose’s networking support is experimental, and must be explicitly enabled with the `docker-compose --x-networking` flag. + +Compose sets up a single default [network](http://TODO/docker-networking-docs) for your app. Each container for a service joins the default network and is both *reachable* by other containers on that network, and *discoverable* by them at a hostname identical to the service's name. + +> **Note:** Your app's network is given the same name as the "project name", which is based on the name of the directory it lives in. See the [CLI docs](cli.md#p-project-name-name) for how to override it. + +For example, suppose your app is in a directory called `myapp`, and your `docker-compose.yml` looks like this: + + web: + build: . + ports: + - "8000:8000" + db: + image: postgres + +When you run `docker-compose --x-networking up`, the following happens: + +1. A network called `myapp` is created. +2. A container is created using `web`'s configuration. It joins the network `myapp` under the name `web`. +3. A container is created using `db`'s configuration. It joins the network `myapp` under the name `db`. + +Each container can now look up the hostname `web` or `db` and get back the appropriate container's IP address. For example, `web`'s application code could connect to the URL `postgres://db:5432` and start using the Postgres database. + +Because `web` explicitly maps a port, it's also accessible from the outside world via port 8000 on your Docker host's network interface. + +## Updating containers + +If you make a configuration change to a service and run `docker-compose up` to update it, the old container will be removed and the new one will join the network under a different IP address but the same name. Running containers will be able to look up that name and connect to the new address, but the old address will stop working. + +If any containers have connections open to the old container, they will be closed. It is a container's responsibility to detect this condition, look up the name again and reconnect. + +## Configure how services are published + +By default, containers for each service are published on the network with the same name as the service. If you want to change the name, or stop containers from being discoverable at all, you can use the `hostname` option: + + web: + build: . + hostname: "my-web-application" + +This will also change the hostname inside the container, so that the `hostname` command will return `my-web-application`. + +## Scaling services + +If you create multiple containers for a service with `docker-compose scale`, each container will join the network with the same name. For example, if you run `docker-compose scale web=3`, then 3 containers will join the network under the name `web`. Inside any container on the network, looking up the name `web` will return the IP address of one of them, but Docker and Compose do not provide any guarantees about which one. + +This limitation will be addressed in a future version of Compose, where a load balancer will join under the service name and balance traffic between the service's containers in a configurable manner. + +## Links + +Docker links are a one-way, single-host communication system. They should now be considered deprecated, and you should update your app to use networking instead. In the majority of cases, this will simply involve removing the `links` sections from your `docker-compose.yml`. + +## Specifying the network driver + +By default, Compose uses the `bridge` driver when creating the app’s network. The Docker Engine provides one other driver out-of-the-box: `overlay`, which implements secure communication between containers on different hosts (see the next section for how to set up and use the `overlay` driver). Docker also allows you to install [custom network drivers](http://TODO/custom-driver-docs). + +You can specify which one to use with the `--x-network-driver` flag: + + $ docker-compose --x-networking --x-network-driver=overlay up + +## Multi-host networking + +(TODO: talk about Swarm and the overlay driver) + +## Custom container network modes + +Compose allows you to specify a custom network mode for a service with the `net` option - for example, `net: "host"` specifies that its containers should use the same network namespace as the Docker host, and `net: "none"` specifies that they should have no networking capabilities. + +If a service specifies the `net` option, its containers will *not* join the app’s network and will not be able to communicate with other services in the app. + +If *all* services in an app specify the `net` option, a network will not be created at all. diff --git a/docs/rails.md b/docs/rails.md index a33cac26edb..9801ef7419f 100644 --- a/docs/rails.md +++ b/docs/rails.md @@ -37,7 +37,7 @@ Next, create a bootstrap `Gemfile` which just loads Rails. It'll be overwritten source 'https://rubygems.org' gem 'rails', '4.2.0' -Finally, `docker-compose.yml` is where the magic happens. This file describes the services that comprise your app (a database and a web app), how to get each one's Docker image (the database just runs on a pre-made PostgreSQL image, and the web app is built from the current directory), and the configuration needed to link them together and expose the web app's port. +Finally, `docker-compose.yml` is where the magic happens. This file describes the services that comprise your app (a database and a web app), how to get each one's Docker image (the database just runs on a pre-made PostgreSQL image, and the web app is built from the current directory), and the configuration needed to expose the web app's port. db: image: postgres @@ -48,8 +48,6 @@ Finally, `docker-compose.yml` is where the magic happens. This file describes th - .:/myapp ports: - "3000:3000" - links: - - db ### Build the project diff --git a/docs/wordpress.md b/docs/wordpress.md index 8c1f5b0acb2..5c9bcdbd9f7 100644 --- a/docs/wordpress.md +++ b/docs/wordpress.md @@ -46,8 +46,6 @@ and a separate MySQL instance: command: php -S 0.0.0.0:8000 -t /code ports: - "8000:8000" - links: - - db volumes: - .:/code db: diff --git a/requirements.txt b/requirements.txt index 4f2ea9d14fe..daaaa950262 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ PyYAML==3.10 -docker-py==1.4.0 +docker-py==1.5.0 dockerpty==0.3.4 docopt==0.6.1 enum34==1.0.4 diff --git a/script/build-windows.ps1 b/script/build-windows.ps1 index b35fad6f131..6e8a7c5ae7f 100644 --- a/script/build-windows.ps1 +++ b/script/build-windows.ps1 @@ -42,15 +42,14 @@ Get-ChildItem -Recurse -Include *.pyc | foreach ($_) { Remove-Item $_.FullName } virtualenv .\venv # Install dependencies -.\venv\Scripts\pip install pypiwin32==219 -.\venv\Scripts\pip install -r requirements.txt -.\venv\Scripts\pip install --no-deps . - # TODO: pip warns when installing from a git sha, so we need to set ErrorAction to # 'Continue'. See # https://github.com/pypa/pip/blob/fbc4b7ae5fee00f95bce9ba4b887b22681327bb1/pip/vcs/git.py#L77 # This can be removed once pyinstaller 3.x is released and we upgrade $ErrorActionPreference = "Continue" +.\venv\Scripts\pip install pypiwin32==219 +.\venv\Scripts\pip install -r requirements.txt +.\venv\Scripts\pip install --no-deps . .\venv\Scripts\pip install --allow-external pyinstaller -r requirements-build.txt # Build binary diff --git a/script/ci b/script/ci index 58144ea3b90..12dc3c473e9 100755 --- a/script/ci +++ b/script/ci @@ -6,7 +6,9 @@ # $ docker build -t "$TAG" . # $ docker run --rm --volume="/var/run/docker.sock:/var/run/docker.sock" --volume="$(pwd)/.git:/code/.git" -e "TAG=$TAG" --entrypoint="script/ci" "$TAG" -set -e +set -ex + +docker version export DOCKER_VERSIONS=all export DOCKER_DAEMON_ARGS="--storage-driver=overlay" diff --git a/setup.py b/setup.py index 0313fbd052d..4020122b157 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ def find_version(*file_paths): 'requests >= 2.6.1, < 2.8', 'texttable >= 0.8.1, < 0.9', 'websocket-client >= 0.32.0, < 1.0', - 'docker-py >= 1.4.0, < 2', + 'docker-py >= 1.5.0, < 2', 'dockerpty >= 0.3.4, < 0.4', 'six >= 1.3.0, < 2', 'jsonschema >= 2.5.1, < 3', diff --git a/tests/integration/cli_test.py b/tests/integration/cli_test.py index 3774eb88e63..78519d14180 100644 --- a/tests/integration/cli_test.py +++ b/tests/integration/cli_test.py @@ -10,6 +10,7 @@ from .. import mock from .testcases import DockerClientTestCase from compose.cli.command import get_project +from compose.cli.docker_client import docker_client from compose.cli.errors import UserError from compose.cli.main import TopLevelCommand from compose.project import NoSuchService @@ -185,6 +186,51 @@ def test_up_attached(self): set(self.project.containers()) ) + def test_up_without_networking(self): + self.require_engine_version("1.9") + + self.command.base_dir = 'tests/fixtures/links-composefile' + self.command.dispatch(['up', '-d'], None) + client = docker_client(version='1.21') + + networks = client.networks(names=[self.project.name]) + self.assertEqual(len(networks), 0) + + for service in self.project.get_services(): + containers = service.containers() + self.assertEqual(len(containers), 1) + self.assertNotEqual(containers[0].get('Config.Hostname'), service.name) + + web_container = self.project.get_service('web').containers()[0] + self.assertTrue(web_container.get('HostConfig.Links')) + + def test_up_with_networking(self): + self.require_engine_version("1.9") + + self.command.base_dir = 'tests/fixtures/links-composefile' + self.command.dispatch(['--x-networking', 'up', '-d'], None) + client = docker_client(version='1.21') + + services = self.project.get_services() + + networks = client.networks(names=[self.project.name]) + for n in networks: + self.addCleanup(client.remove_network, n['id']) + self.assertEqual(len(networks), 1) + self.assertEqual(networks[0]['driver'], 'bridge') + + network = client.inspect_network(networks[0]['id']) + self.assertEqual(len(network['containers']), len(services)) + + for service in services: + containers = service.containers() + self.assertEqual(len(containers), 1) + self.assertIn(containers[0].id, network['containers']) + self.assertEqual(containers[0].get('Config.Hostname'), service.name) + + web_container = self.project.get_service('web').containers()[0] + self.assertFalse(web_container.get('HostConfig.Links')) + def test_up_with_links(self): self.command.base_dir = 'tests/fixtures/links-composefile' self.command.dispatch(['up', '-d', 'web'], None) diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index 26a0a108a1c..a412fb04fb7 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -2,6 +2,8 @@ from __future__ import unicode_literals from docker import errors +from docker.utils import version_lt +from pytest import skip from .. import unittest from compose.cli.docker_client import docker_client @@ -73,3 +75,12 @@ def check_build(self, *args, **kwargs): kwargs.setdefault('rm', True) build_output = self.client.build(*args, **kwargs) stream_output(build_output, open('/dev/null', 'w')) + + def require_engine_version(self, minimum): + # Drop '-dev' or '-rcN' suffix + engine = self.client.version()['Version'].split('-', 1)[0] + if version_lt(engine, minimum): + skip( + "Engine version is too low ({} < {})" + .format(engine, minimum) + ) diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 84ede755312..c5e1a9fb060 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -203,6 +203,26 @@ def test_split_domainname_weird(self): self.assertEqual(opts['hostname'], 'name.sub', 'hostname') self.assertEqual(opts['domainname'], 'domain.tld', 'domainname') + def test_no_default_hostname_when_not_using_networking(self): + service = Service( + 'foo', + image='foo', + use_networking=False, + client=self.mock_client, + ) + opts = service._get_container_create_options({'image': 'foo'}, 1) + self.assertIsNone(opts.get('hostname')) + + def test_hostname_defaults_to_service_name_when_using_networking(self): + service = Service( + 'foo', + image='foo', + use_networking=True, + client=self.mock_client, + ) + opts = service._get_container_create_options({'image': 'foo'}, 1) + self.assertEqual(opts['hostname'], 'foo') + def test_get_container_create_options_with_name_option(self): service = Service( 'foo', diff --git a/tox.ini b/tox.ini index dbf639201b1..f05c5ed260c 100644 --- a/tox.ini +++ b/tox.ini @@ -11,9 +11,10 @@ passenv = setenv = HOME=/tmp deps = + -rrequirements.txt -rrequirements-dev.txt commands = - py.test -v \ + py.test -v -rxs \ --cov=compose \ --cov-report html \ --cov-report term \