From 504b67129365f664551a1b7dd265d50a8b919841 Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Sat, 18 Jun 2022 14:39:38 +0200 Subject: [PATCH 01/14] Rewrite the docker_host_info module. --- plugins/modules/docker_host_info.py | 59 ++++++++++++++--------------- 1 file changed, 29 insertions(+), 30 deletions(-) diff --git a/plugins/modules/docker_host_info.py b/plugins/modules/docker_host_info.py index dd09e3345..d59986144 100644 --- a/plugins/modules/docker_host_info.py +++ b/plugins/modules/docker_host_info.py @@ -94,15 +94,13 @@ type: bool default: no extends_documentation_fragment: -- community.docker.docker -- community.docker.docker.docker_py_1_documentation +- community.docker.docker.api_documentation author: - Piotr Wojciechowski (@WojciechowskiPiotr) requirements: - - "L(Docker SDK for Python,https://docker-py.readthedocs.io/en/stable/) >= 1.10.0" - "Docker API >= 1.25" ''' @@ -205,21 +203,17 @@ from ansible.module_utils.common.text.converters import to_native -from ansible_collections.community.docker.plugins.module_utils.common import ( +from ansible_collections.community.docker.plugins.module_utils.common_api import ( AnsibleDockerClient, RequestException, ) -try: - from docker.errors import DockerException, APIError -except ImportError: - # Missing Docker SDK for Python handled in ansible.module_utils.docker.common - pass - from ansible_collections.community.docker.plugins.module_utils.util import ( DockerBaseClass, clean_dict_booleans_for_docker_api, ) +from ansible_collections.community.docker.plugins.module_utils._api.errors import DockerException, APIError +from ansible_collections.community.docker.plugins.module_utils._api.utils.utils import convert_filters class DockerHostManager(DockerBaseClass): @@ -275,25 +269,37 @@ def get_docker_items_list(self, docker_object=None, filters=None, verbose=False) filter_arg['filters'] = filters try: if docker_object == 'containers': - items = self.client.containers(**filter_arg) + params = { + 'limit': -1, + 'all': 0, + 'size': 0, + 'trunc_cmd': 0, + 'filters': convert_filters(filters) if filters else None, + } + items = self.client.get_json("/containers/json", params=params) elif docker_object == 'networks': - items = self.client.networks(**filter_arg) + params = { + 'filters': convert_filters(filters or {}) + } + items = self.client.get_json("/networks", params=params) elif docker_object == 'images': - items = self.client.images(**filter_arg) + params = { + 'only_ids': 0, + 'all': 0, + 'filters': convert_filters(filters) if filters else None, + } + items = self.client.get_json("/images/json", params=params) elif docker_object == 'volumes': - items = self.client.volumes(**filter_arg) + params = { + 'filters': convert_filters(filters) if filters else None, + } + items = self.client.get_json('/volumes', params=params) + items = items['Volumes'] except APIError as exc: - self.client.fail("Error inspecting docker host for object '%s': %s" % - (docker_object, to_native(exc))) + self.client.fail("Error inspecting docker host for object '%s': %s" % (docker_object, to_native(exc))) if self.verbose_output: - if docker_object != 'volumes': - return items - else: - return items['Volumes'] - - if docker_object == 'volumes': - items = items['Volumes'] + return items for item in items: item_record = dict() @@ -329,16 +335,9 @@ def main(): verbose_output=dict(type='bool', default=False), ) - option_minimal_versions = dict( - network_filters=dict(docker_py_version='2.0.2'), - disk_usage=dict(docker_py_version='2.2.0'), - ) - client = AnsibleDockerClient( argument_spec=argument_spec, supports_check_mode=True, - min_docker_version='1.10.0', - option_minimal_versions=option_minimal_versions, fail_results=dict( can_talk_to_docker=False, ), From 9b275161dd260a9843df59b4dce53affe02b5881 Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Sat, 18 Jun 2022 17:59:55 +0200 Subject: [PATCH 02/14] Add changelog. --- changelogs/fragments/387-docker-api.yml | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 changelogs/fragments/387-docker-api.yml diff --git a/changelogs/fragments/387-docker-api.yml b/changelogs/fragments/387-docker-api.yml new file mode 100644 index 000000000..02c762f66 --- /dev/null +++ b/changelogs/fragments/387-docker-api.yml @@ -0,0 +1,8 @@ +major_changes: + - "The collection now contains vendored code from the Docker SDK for Python to talk to the Docker daemon. + Modules and plugins using this code no longer need the Docker SDK for Python installed on the machine + the module resp. plugin is running on + (https://github.com/ansible-collections/community.docker/pull/387)." + - "docker_host_info - no longer uses the Docker SDK for Python. It requires ``requests`` to be installed, + and depending on the features used has some more requirements. If the Docker SDK for Python is installed, + these requirements are likely met (https://github.com/ansible-collections/community.docker/pull/387)." From 0374f0e8b20a56d801172d1b511d09fecd875e30 Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Sat, 18 Jun 2022 21:28:59 +0200 Subject: [PATCH 03/14] Convert docker_container_info, docker_image_info, docker_network_info, and docker_volume_info. --- plugins/module_utils/common_api.py | 193 +++++++++++++++++++++++ plugins/modules/docker_container_info.py | 12 +- plugins/modules/docker_image_info.py | 27 ++-- plugins/modules/docker_network_info.py | 13 +- plugins/modules/docker_volume_info.py | 17 +- 5 files changed, 216 insertions(+), 46 deletions(-) diff --git a/plugins/module_utils/common_api.py b/plugins/module_utils/common_api.py index da844d441..a1f3f189a 100644 --- a/plugins/module_utils/common_api.py +++ b/plugins/module_utils/common_api.py @@ -580,3 +580,196 @@ def report_warnings(self, result, warnings_key=None): self.module.warn('Docker warning: {0}'.format(warning)) elif isinstance(result, string_types) and result: self.module.warn('Docker warning: {0}'.format(result)) + + def get_container_by_id(self, container_id): + try: + self.log("Inspecting container Id %s" % container_id) + result = self.get_json('/containers/{0}/json', container_id) + self.log("Completed container inspection") + return result + except NotFound as dummy: + return None + except Exception as exc: + self.fail("Error inspecting container: %s" % exc) + + def get_container(self, name=None): + ''' + Lookup a container and return the inspection results. + ''' + if name is None: + return None + + search_name = name + if not name.startswith('/'): + search_name = '/' + name + + result = None + try: + params = { + 'limit': -1, + 'all': 1, + 'size': 0, + 'trunc_cmd': 0, + } + containers = self.get_json("/containers/json", params=params) + for container in containers: + self.log("testing container: %s" % (container['Names'])) + if isinstance(container['Names'], list) and search_name in container['Names']: + result = container + break + if container['Id'].startswith(name): + result = container + break + if container['Id'] == name: + result = container + break + except SSLError as exc: + self._handle_ssl_error(exc) + except Exception as exc: + self.fail("Error retrieving container list: %s" % exc) + + if result is None: + return None + + return self.get_container_by_id(result['Id']) + + def get_network(self, name=None, network_id=None): + ''' + Lookup a network and return the inspection results. + ''' + if name is None and network_id is None: + return None + + result = None + + if network_id is None: + try: + networks = self.get_json("/networks") + for network in networks: + self.log("testing network: %s" % (network['Name'])) + if name == network['Name']: + result = network + break + if network['Id'].startswith(name): + result = network + break + except SSLError as exc: + self._handle_ssl_error(exc) + except Exception as exc: + self.fail("Error retrieving network list: %s" % exc) + + if result is not None: + network_id = result['Id'] + + if network_id is not None: + try: + self.log("Inspecting network Id %s" % network_id) + result = self.get_json('/networks/{0}', network_id) + self.log("Completed network inspection") + except NotFound as dummy: + return None + except Exception as exc: + self.fail("Error inspecting network: %s" % exc) + + return result + + def _image_lookup(self, name, tag): + ''' + Including a tag in the name parameter sent to the Docker SDK for Python images method + does not work consistently. Instead, get the result set for name and manually check + if the tag exists. + ''' + try: + params = { + 'only_ids': 0, + 'all': 0, + } + if LooseVersion(self.api_version) < LooseVersion('1.25'): + # only use "filter" on API 1.24 and under, as it is deprecated + params['filter'] = name + else: + params['filters'] = convert_filters({'reference': name}) + images = self.get_json("/images/json", params=params) + except Exception as exc: + self.fail("Error searching for image %s - %s" % (name, str(exc))) + if tag: + lookup = "%s:%s" % (name, tag) + lookup_digest = "%s@%s" % (name, tag) + response = images + images = [] + for image in response: + tags = image.get('RepoTags') + digests = image.get('RepoDigests') + if (tags and lookup in tags) or (digests and lookup_digest in digests): + images = [image] + break + return images + + def find_image(self, name, tag): + ''' + Lookup an image (by name and tag) and return the inspection results. + ''' + if not name: + return None + + self.log("Find image %s:%s" % (name, tag)) + images = self._image_lookup(name, tag) + if not images: + # In API <= 1.20 seeing 'docker.io/' as the name of images pulled from docker hub + registry, repo_name = auth.resolve_repository_name(name) + if registry == 'docker.io': + # If docker.io is explicitly there in name, the image + # isn't found in some cases (#41509) + self.log("Check for docker.io image: %s" % repo_name) + images = self._image_lookup(repo_name, tag) + if not images and repo_name.startswith('library/'): + # Sometimes library/xxx images are not found + lookup = repo_name[len('library/'):] + self.log("Check for docker.io image: %s" % lookup) + images = self._image_lookup(lookup, tag) + if not images: + # Last case for some Docker versions: if docker.io wasn't there, + # it can be that the image wasn't found either + # (https://github.com/ansible/ansible/pull/15586) + lookup = "%s/%s" % (registry, repo_name) + self.log("Check for docker.io image: %s" % lookup) + images = self._image_lookup(lookup, tag) + if not images and '/' not in repo_name: + # This seems to be happening with podman-docker + # (https://github.com/ansible-collections/community.docker/issues/291) + lookup = "%s/library/%s" % (registry, repo_name) + self.log("Check for docker.io image: %s" % lookup) + images = self._image_lookup(lookup, tag) + + if len(images) > 1: + self.fail("Registry returned more than one result for %s:%s" % (name, tag)) + + if len(images) == 1: + try: + return self.get_json('/images/{0}/json', images[0]['Id']) + except NotFound: + self.log("Image %s:%s not found." % (name, tag)) + return None + except Exception as exc: + self.fail("Error inspecting image %s:%s - %s" % (name, tag, str(exc))) + + self.log("Image %s:%s not found." % (name, tag)) + return None + + def find_image_by_id(self, image_id, accept_missing_image=False): + ''' + Lookup an image (by ID) and return the inspection results. + ''' + if not image_id: + return None + + self.log("Find image %s (by ID)" % image_id) + try: + return self.get_json('/images/{0}/json', image_id) + except NotFound as exc: + if not accept_missing_image: + self.fail("Error inspecting image ID %s - %s" % (image_id, str(exc))) + self.log("Image %s not found." % image_id) + return None + except Exception as exc: + self.fail("Error inspecting image ID %s - %s" % (image_id, str(exc))) diff --git a/plugins/modules/docker_container_info.py b/plugins/modules/docker_container_info.py index b9f229205..f75a1a174 100644 --- a/plugins/modules/docker_container_info.py +++ b/plugins/modules/docker_container_info.py @@ -27,8 +27,7 @@ type: str required: yes extends_documentation_fragment: -- community.docker.docker -- community.docker.docker.docker_py_1_documentation +- community.docker.docker.api_documentation author: @@ -106,16 +105,11 @@ from ansible.module_utils.common.text.converters import to_native -try: - from docker.errors import DockerException -except ImportError: - # missing Docker SDK for Python handled in ansible.module_utils.docker.common - pass - -from ansible_collections.community.docker.plugins.module_utils.common import ( +from ansible_collections.community.docker.plugins.module_utils.common_api import ( AnsibleDockerClient, RequestException, ) +from ansible_collections.community.docker.plugins.module_utils._api.errors import DockerException def main(): diff --git a/plugins/modules/docker_image_info.py b/plugins/modules/docker_image_info.py index e237baa74..2dcc95f14 100644 --- a/plugins/modules/docker_image_info.py +++ b/plugins/modules/docker_image_info.py @@ -36,12 +36,10 @@ elements: str extends_documentation_fragment: -- community.docker.docker -- community.docker.docker.docker_py_1_documentation +- community.docker.docker.api_documentation requirements: - - "L(Docker SDK for Python,https://docker-py.readthedocs.io/en/stable/) >= 1.8.0" - "Docker API >= 1.25" author: @@ -167,14 +165,7 @@ from ansible.module_utils.common.text.converters import to_native -try: - from docker import utils - from docker.errors import DockerException, NotFound -except ImportError: - # missing Docker SDK for Python handled in ansible.module_utils.docker.common - pass - -from ansible_collections.community.docker.plugins.module_utils.common import ( +from ansible_collections.community.docker.plugins.module_utils.common_api import ( AnsibleDockerClient, RequestException, ) @@ -182,6 +173,8 @@ DockerBaseClass, is_image_name_id, ) +from ansible_collections.community.docker.plugins.module_utils._api.errors import DockerException, NotFound +from ansible_collections.community.docker.plugins.module_utils._api.utils.utils import parse_repository_tag class ImageManager(DockerBaseClass): @@ -221,7 +214,7 @@ def get_facts(self): self.log('Fetching image %s (ID)' % (name)) image = self.client.find_image_by_id(name, accept_missing_image=True) else: - repository, tag = utils.parse_repository_tag(name) + repository, tag = parse_repository_tag(name) if not tag: tag = 'latest' self.log('Fetching image %s:%s' % (repository, tag)) @@ -232,12 +225,16 @@ def get_facts(self): def get_all_images(self): results = [] - images = self.client.images() + params = { + 'only_ids': 0, + 'all': 0, + } + images = self.client.get_json("/images/json", params=params) for image in images: try: - inspection = self.client.inspect_image(image['Id']) + inspection = self.client.get_json('/images/{0}/json', image['Id']) except NotFound: - pass + inspection = None except Exception as exc: self.fail("Error inspecting image %s - %s" % (image['Id'], to_native(exc))) results.append(inspection) diff --git a/plugins/modules/docker_network_info.py b/plugins/modules/docker_network_info.py index 202c7617d..bc4e4210e 100644 --- a/plugins/modules/docker_network_info.py +++ b/plugins/modules/docker_network_info.py @@ -27,15 +27,13 @@ type: str required: yes extends_documentation_fragment: -- community.docker.docker -- community.docker.docker.docker_py_1_documentation +- community.docker.docker.api_documentation author: - "Dave Bendit (@DBendit)" requirements: - - "L(Docker SDK for Python,https://docker-py.readthedocs.io/en/stable/) >= 1.8.0" - "Docker API >= 1.25" ''' @@ -102,16 +100,11 @@ from ansible.module_utils.common.text.converters import to_native -try: - from docker.errors import DockerException -except ImportError: - # missing Docker SDK for Python handled in ansible.module_utils.docker.common - pass - -from ansible_collections.community.docker.plugins.module_utils.common import ( +from ansible_collections.community.docker.plugins.module_utils.common_api import ( AnsibleDockerClient, RequestException, ) +from ansible_collections.community.docker.plugins.module_utils._api.errors import DockerException def main(): diff --git a/plugins/modules/docker_volume_info.py b/plugins/modules/docker_volume_info.py index 2f9993a77..4afab3ae5 100644 --- a/plugins/modules/docker_volume_info.py +++ b/plugins/modules/docker_volume_info.py @@ -23,15 +23,14 @@ - volume_name extends_documentation_fragment: -- community.docker.docker -- community.docker.docker.docker_py_1_documentation +- community.docker.docker.api_documentation author: - Felix Fontein (@felixfontein) requirements: - - "L(Docker SDK for Python,https://docker-py.readthedocs.io/en/stable/) >= 1.8.0" + - "Docker API >= 1.25" ''' EXAMPLES = ''' @@ -78,21 +77,16 @@ from ansible.module_utils.common.text.converters import to_native -try: - from docker.errors import DockerException, NotFound -except ImportError: - # missing Docker SDK for Python handled in ansible.module_utils.docker.common - pass - -from ansible_collections.community.docker.plugins.module_utils.common import ( +from ansible_collections.community.docker.plugins.module_utils.common_api import ( AnsibleDockerClient, RequestException, ) +from ansible_collections.community.docker.plugins.module_utils._api.errors import DockerException, NotFound def get_existing_volume(client, volume_name): try: - return client.inspect_volume(volume_name) + return client.get_json('/volumes/{0}', volume_name) except NotFound as dummy: return None except Exception as exc: @@ -107,7 +101,6 @@ def main(): client = AnsibleDockerClient( argument_spec=argument_spec, supports_check_mode=True, - min_docker_version='1.8.0', ) try: From ad74f59c3cc91ee187203d44fb174047f6237654 Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Sun, 19 Jun 2022 21:14:12 +0200 Subject: [PATCH 04/14] Adjust docker_volume. --- plugins/modules/docker_volume.py | 46 ++++++++----------- .../plugins/modules/test_docker_volume.py | 36 --------------- 2 files changed, 18 insertions(+), 64 deletions(-) delete mode 100644 tests/unit/plugins/modules/test_docker_volume.py diff --git a/plugins/modules/docker_volume.py b/plugins/modules/docker_volume.py index 90e207091..8a786356c 100644 --- a/plugins/modules/docker_volume.py +++ b/plugins/modules/docker_volume.py @@ -68,15 +68,13 @@ - present extends_documentation_fragment: -- community.docker.docker -- community.docker.docker.docker_py_1_documentation +- community.docker.docker.api_documentation author: - Alex Grönholm (@agronholm) requirements: - - "L(Docker SDK for Python,https://docker-py.readthedocs.io/en/stable/) >= 1.10.0" - "Docker API >= 1.25" ''' @@ -110,14 +108,9 @@ import traceback from ansible.module_utils.common.text.converters import to_native +from ansible.module_utils.six import iteritems, text_type -try: - from docker.errors import DockerException, APIError -except ImportError: - # missing Docker SDK for Python handled in ansible.module_utils.docker.common - pass - -from ansible_collections.community.docker.plugins.module_utils.common import ( +from ansible_collections.community.docker.plugins.module_utils.common_api import ( AnsibleDockerClient, RequestException, ) @@ -125,7 +118,11 @@ DockerBaseClass, DifferenceTracker, ) -from ansible.module_utils.six import iteritems, text_type +from ansible_collections.community.docker.plugins.module_utils._api.errors import ( + APIError, + DockerException, + NotFound, +) class TaskParameters(DockerBaseClass): @@ -173,7 +170,7 @@ def __init__(self, client): def get_existing_volume(self): try: - volumes = self.client.volumes() + volumes = self.client.get_json('/volumes') except APIError as e: self.client.fail(to_native(e)) @@ -221,16 +218,15 @@ def create_volume(self): if not self.existing_volume: if not self.check_mode: try: - params = dict( - driver=self.parameters.driver, - driver_opts=self.parameters.driver_options, - ) - + data = { + 'Name': self.parameters.volume_name, + 'Driver': self.parameters.driver, + 'DriverOpts': self.parameters.driver_options, + } if self.parameters.labels is not None: - params['labels'] = self.parameters.labels - - resp = self.client.create_volume(self.parameters.volume_name, **params) - self.existing_volume = self.client.inspect_volume(resp['Name']) + data['Labels'] = self.parameters.labels + resp = self.client.post_json_to_json('/volumes/create', data=data) + self.existing_volume = self.client.get_json('/volumes/{0}', resp['Name']) except APIError as e: self.client.fail(to_native(e)) @@ -241,7 +237,7 @@ def remove_volume(self): if self.existing_volume: if not self.check_mode: try: - self.client.remove_volume(self.parameters.volume_name) + self.client.delete_call('/volumes/{0}', self.parameters.volume_name) except APIError as e: self.client.fail(to_native(e)) @@ -286,16 +282,10 @@ def main(): debug=dict(type='bool', default=False) ) - option_minimal_versions = dict( - labels=dict(docker_py_version='1.10.0'), - ) - client = AnsibleDockerClient( argument_spec=argument_spec, supports_check_mode=True, - min_docker_version='1.10.0', # "The docker server >= 1.9.0" - option_minimal_versions=option_minimal_versions, ) try: diff --git a/tests/unit/plugins/modules/test_docker_volume.py b/tests/unit/plugins/modules/test_docker_volume.py deleted file mode 100644 index 6bce3f380..000000000 --- a/tests/unit/plugins/modules/test_docker_volume.py +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright (c) 2018 Ansible Project -# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) - -from __future__ import (absolute_import, division, print_function) -__metaclass__ = type - -import json - -import pytest - -from ansible_collections.community.docker.plugins.modules import docker_volume -from ansible_collections.community.docker.plugins.module_utils import common - -pytestmark = pytest.mark.usefixtures('patch_ansible_module') - -TESTCASE_DOCKER_VOLUME = [ - { - 'name': 'daemon_config', - 'state': 'present' - } -] - - -@pytest.mark.parametrize('patch_ansible_module', TESTCASE_DOCKER_VOLUME, indirect=['patch_ansible_module']) -def test_create_volume_on_invalid_docker_version(mocker, capfd): - mocker.patch.object(common, 'HAS_DOCKER_PY', True) - mocker.patch.object(common, 'docker_version', '1.8.0') - - with pytest.raises(SystemExit): - docker_volume.main() - - out, dummy = capfd.readouterr() - results = json.loads(out) - assert results['failed'] - assert 'Error: Docker SDK for Python version is 1.8.0 ' in results['msg'] - assert 'Minimum version required is 1.10.0.' in results['msg'] From f871b417004072dbc0082fb17059966c0a2d17f0 Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Sun, 19 Jun 2022 21:37:59 +0200 Subject: [PATCH 05/14] Adjust docker_network. --- plugins/modules/docker_network.py | 101 ++++++++++++++---------------- 1 file changed, 46 insertions(+), 55 deletions(-) diff --git a/plugins/modules/docker_network.py b/plugins/modules/docker_network.py index 622d52ea1..32492c920 100644 --- a/plugins/modules/docker_network.py +++ b/plugins/modules/docker_network.py @@ -146,8 +146,7 @@ type: bool extends_documentation_fragment: -- community.docker.docker -- community.docker.docker.docker_py_1_documentation +- community.docker.docker.api_documentation notes: @@ -165,7 +164,6 @@ - "Dave Bendit (@DBendit)" requirements: - - "L(Docker SDK for Python,https://docker-py.readthedocs.io/en/stable/) >= 1.10.0" - "Docker API >= 1.25" ''' @@ -254,27 +252,16 @@ from ansible.module_utils.common.text.converters import to_native -from ansible_collections.community.docker.plugins.module_utils.version import LooseVersion - -from ansible_collections.community.docker.plugins.module_utils.common import ( +from ansible_collections.community.docker.plugins.module_utils.common_api import ( AnsibleDockerClient, RequestException, - docker_version, ) from ansible_collections.community.docker.plugins.module_utils.util import ( DockerBaseClass, DifferenceTracker, clean_dict_booleans_for_docker_api, ) - -try: - from docker import utils - from docker.errors import DockerException - if LooseVersion(docker_version) >= LooseVersion('2.0.0'): - from docker.types import IPAMPool, IPAMConfig -except Exception: - # missing Docker SDK for Python handled in ansible.module_utils.docker.common - pass +from ansible_collections.community.docker.plugins.module_utils._api.errors import DockerException class TaskParameters(DockerBaseClass): @@ -496,45 +483,48 @@ def has_different_config(self, net): def create_network(self): if not self.existing_network: - params = dict( - driver=self.parameters.driver, - options=self.parameters.driver_options, - ) + data = { + 'Name': self.parameters.name, + 'Driver': self.parameters.driver, + 'Options': self.parameters.driver_options, + 'IPAM': None, + 'CheckDuplicate': None, + } + + if self.parameters.enable_ipv6: + data['EnableIPv6'] = True + if self.parameters.internal: + data['Internal'] = True + if self.parameters.scope is not None: + data['Scope'] = self.parameters.scope + if self.parameters.attachable is not None: + data['Attachable'] = self.parameters.attachable + if self.parameters.labels is not None: + data["Labels"] = self.parameters.labels ipam_pools = [] if self.parameters.ipam_config: for ipam_pool in self.parameters.ipam_config: - if LooseVersion(docker_version) >= LooseVersion('2.0.0'): - ipam_pools.append(IPAMPool(**ipam_pool)) - else: - ipam_pools.append(utils.create_ipam_pool(**ipam_pool)) + ipam_pools.append({ + 'Subnet': ipam_pool['subnet'], + 'IPRange': ipam_pool['iprange'], + 'Gateway': ipam_pool['gateway'], + 'AuxiliaryAddresses': ipam_pool['aux_addresses'], + }) if self.parameters.ipam_driver or self.parameters.ipam_driver_options or ipam_pools: - # Only add ipam parameter if a driver was specified or if IPAM parameters - # were specified. Leaving this parameter away can significantly speed up + # Only add IPAM if a driver was specified or if IPAM parameters were + # specified. Leaving this parameter away can significantly speed up # creation; on my machine creation with this option needs ~15 seconds, # and without just a few seconds. - if LooseVersion(docker_version) >= LooseVersion('2.0.0'): - params['ipam'] = IPAMConfig(driver=self.parameters.ipam_driver, - pool_configs=ipam_pools, - options=self.parameters.ipam_driver_options) - else: - params['ipam'] = utils.create_ipam_config(driver=self.parameters.ipam_driver, - pool_configs=ipam_pools) - - if self.parameters.enable_ipv6 is not None: - params['enable_ipv6'] = self.parameters.enable_ipv6 - if self.parameters.internal is not None: - params['internal'] = self.parameters.internal - if self.parameters.scope is not None: - params['scope'] = self.parameters.scope - if self.parameters.attachable is not None: - params['attachable'] = self.parameters.attachable - if self.parameters.labels: - params['labels'] = self.parameters.labels + data['IPAM'] = { + 'Driver': self.parameters.ipam_driver, + 'Config': ipam_pools or [], + 'Options': self.parameters.ipam_driver_options, + } if not self.check_mode: - resp = self.client.create_network(self.parameters.name, **params) + resp = self.client.post_json_to_json('/networks/create', data=data) self.client.report_warnings(resp, ['Warning']) self.existing_network = self.client.get_network(network_id=resp['Id']) self.results['actions'].append("Created network %s with driver %s" % (self.parameters.name, self.parameters.driver)) @@ -544,7 +534,7 @@ def remove_network(self): if self.existing_network: self.disconnect_all_containers() if not self.check_mode: - self.client.remove_network(self.parameters.name) + self.client.delete_call('/networks/{0}', self.parameters.name) self.results['actions'].append("Removed network %s" % (self.parameters.name,)) self.results['changed'] = True @@ -557,12 +547,14 @@ def connect_containers(self): for name in self.parameters.connected: if not self.is_container_connected(name): if not self.check_mode: - self.client.connect_container_to_network(name, self.parameters.name) + data = { + "Container": name, + "EndpointConfig": None, + } + self.client.post_json('/networks/{0}/connect', self.parameters.name, data=data) self.results['actions'].append("Connected container %s" % (name,)) self.results['changed'] = True - self.diff_tracker.add('connected.{0}'.format(name), - parameter=True, - active=False) + self.diff_tracker.add('connected.{0}'.format(name), parameter=True, active=False) def disconnect_missing(self): if not self.existing_network: @@ -584,7 +576,8 @@ def disconnect_all_containers(self): def disconnect_container(self, container_name): if not self.check_mode: - self.client.disconnect_container_from_network(container_name, self.parameters.name) + data = {"Container": container_name} + self.client.post_json('/networks/{0}/disconnect', self.parameters.name, data=data) self.results['actions'].append("Disconnected container %s" % (container_name,)) self.results['changed'] = True self.diff_tracker.add('connected.{0}'.format(container_name), @@ -648,15 +641,13 @@ def main(): ) option_minimal_versions = dict( - scope=dict(docker_py_version='2.6.0', docker_api_version='1.30'), - attachable=dict(docker_py_version='2.0.0', docker_api_version='1.26'), - ipam_driver_options=dict(docker_py_version='2.0.0'), + scope=dict(docker_api_version='1.30'), + attachable=dict(docker_api_version='1.26'), ) client = AnsibleDockerClient( argument_spec=argument_spec, supports_check_mode=True, - min_docker_version='1.10.0', # "The docker server >= 1.10.0" option_minimal_versions=option_minimal_versions, ) From 42254d3e7cc604d59e412d64b06a304a9c27a346 Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Sun, 19 Jun 2022 22:40:57 +0200 Subject: [PATCH 06/14] Remove no longer needed conditionals on docker_py_version from tests. --- .../docker_container_info/tasks/main.yml | 4 +-- .../targets/docker_host_info/tasks/main.yml | 4 +-- .../docker_host_info/tasks/test_host_info.yml | 26 ++----------------- .../targets/docker_image_info/tasks/main.yml | 4 +-- .../targets/docker_network/tasks/main.yml | 4 +-- .../docker_network/tasks/tests/ipam.yml | 7 ----- .../docker_network/tasks/tests/options.yml | 10 ------- .../docker_network_info/tasks/main.yml | 4 +-- .../targets/docker_volume/tasks/main.yml | 4 +-- .../targets/docker_volume_info/tasks/main.yml | 5 ++-- 10 files changed, 16 insertions(+), 56 deletions(-) diff --git a/tests/integration/targets/docker_container_info/tasks/main.yml b/tests/integration/targets/docker_container_info/tasks/main.yml index 76830d228..0923717a1 100644 --- a/tests/integration/targets/docker_container_info/tasks/main.yml +++ b/tests/integration/targets/docker_container_info/tasks/main.yml @@ -74,7 +74,7 @@ state: absent force_kill: yes - when: docker_py_version is version('1.8.0', '>=') and docker_api_version is version('1.25', '>=') + when: docker_api_version is version('1.25', '>=') - fail: msg="Too old docker / docker-py version to run docker_container_info tests!" - when: not(docker_py_version is version('1.8.0', '>=') and docker_api_version is version('1.25', '>=')) and (ansible_distribution != 'CentOS' or ansible_distribution_major_version|int > 6) + when: not(docker_api_version is version('1.25', '>=')) and (ansible_distribution != 'CentOS' or ansible_distribution_major_version|int > 6) diff --git a/tests/integration/targets/docker_host_info/tasks/main.yml b/tests/integration/targets/docker_host_info/tasks/main.yml index 8d4d8edd5..ad51293f0 100644 --- a/tests/integration/targets/docker_host_info/tasks/main.yml +++ b/tests/integration/targets/docker_host_info/tasks/main.yml @@ -4,7 +4,7 @@ #################################################################### - include_tasks: test_host_info.yml - when: docker_py_version is version('1.10.0', '>=') and docker_api_version is version('1.25', '>=') + when: docker_api_version is version('1.25', '>=') - fail: msg="Too old docker / docker-py version to run docker_host_info tests!" - when: not(docker_py_version is version('1.10.0', '>=') and docker_api_version is version('1.25', '>=')) and (ansible_distribution != 'CentOS' or ansible_distribution_major_version|int > 6) + when: not(docker_api_version is version('1.25', '>=')) and (ansible_distribution != 'CentOS' or ansible_distribution_major_version|int > 6) diff --git a/tests/integration/targets/docker_host_info/tasks/test_host_info.yml b/tests/integration/targets/docker_host_info/tasks/test_host_info.yml index 90dac3ec5..55619c07f 100644 --- a/tests/integration/targets/docker_host_info/tasks/test_host_info.yml +++ b/tests/integration/targets/docker_host_info/tasks/test_host_info.yml @@ -224,7 +224,6 @@ docker_host_info: disk_usage: yes register: output - ignore_errors: yes - name: assert reading docker host facts when docker is running and get disk usage assert: @@ -236,20 +235,12 @@ - 'output.images is not defined' - 'output.disk_usage.LayersSize is number' - 'output.disk_usage.BuilderSize is not defined' - when: docker_py_version is version('2.2.0', '>=') - - assert: - that: - - output is failed - - "('version is ' ~ docker_py_version ~ ' ') in output.msg" - - "'Minimum version required is 2.2.0 ' in output.msg" - when: docker_py_version is version('2.2.0', '<') - name: Get info on Docker host and get disk usage with verbose output docker_host_info: disk_usage: yes verbose_output: yes register: output - ignore_errors: yes - name: assert reading docker host facts when docker is running and get disk usage with verbose output assert: @@ -261,13 +252,6 @@ - 'output.images is not defined' - 'output.disk_usage.LayersSize is number' - 'output.disk_usage.BuilderSize is number' - when: docker_py_version is version('2.2.0', '>=') - - assert: - that: - - output is failed - - "('version is ' ~ docker_py_version ~ ' ') in output.msg" - - "'Minimum version required is 2.2.0 ' in output.msg" - when: docker_py_version is version('2.2.0', '<') - name: Get info on Docker host, disk usage and get all lists together docker_host_info: @@ -275,7 +259,7 @@ containers: yes networks: yes images: yes - disk_usage: "{{ docker_py_version is version('2.2.0', '>=') }}" + disk_usage: yes register: output - name: assert reading docker host facts when docker is running, disk usage and get lists together @@ -290,11 +274,8 @@ - 'output.volumes[0].Mountpoint is not defined' - 'output.images[0].Id is string' - 'output.images[0].ParentId is not defined' - - assert: - that: - 'output.disk_usage.LayersSize is number' - 'output.disk_usage.BuilderSize is not defined' - when: docker_py_version is version('2.2.0', '>=') - name: Get info on Docker host, disk usage and get all lists together with verbose output docker_host_info: @@ -302,7 +283,7 @@ containers: yes networks: yes images: yes - disk_usage: "{{ docker_py_version is version('2.2.0', '>=') }}" + disk_usage: yes verbose_output: yes register: output @@ -318,11 +299,8 @@ - 'output.volumes[0].Mountpoint is string' - 'output.images[0].Id is string' - 'output.images[0].ParentId is string' - - assert: - that: - 'output.disk_usage.LayersSize is number' - 'output.disk_usage.BuilderSize is number' - when: docker_py_version is version('2.2.0', '>=') always: - name: Delete container diff --git a/tests/integration/targets/docker_image_info/tasks/main.yml b/tests/integration/targets/docker_image_info/tasks/main.yml index f14a108b8..559daaefc 100644 --- a/tests/integration/targets/docker_image_info/tasks/main.yml +++ b/tests/integration/targets/docker_image_info/tasks/main.yml @@ -53,7 +53,7 @@ - "docker_test_image_hello_world in result.images[0].RepoTags" - "docker_test_image_alpine in result.images[1].RepoTags" - when: docker_py_version is version('1.8.0', '>=') and docker_api_version is version('1.25', '>=') + when: docker_api_version is version('1.25', '>=') - fail: msg="Too old docker / docker-py version to run docker_image_info tests!" - when: not(docker_py_version is version('1.8.0', '>=') and docker_api_version is version('1.25', '>=')) and (ansible_distribution != 'CentOS' or ansible_distribution_major_version|int > 6) + when: not(docker_api_version is version('1.25', '>=')) and (ansible_distribution != 'CentOS' or ansible_distribution_major_version|int > 6) diff --git a/tests/integration/targets/docker_network/tasks/main.yml b/tests/integration/targets/docker_network/tasks/main.yml index 61a2fd562..d172b1dd1 100644 --- a/tests/integration/targets/docker_network/tasks/main.yml +++ b/tests/integration/targets/docker_network/tasks/main.yml @@ -42,7 +42,7 @@ force: yes loop: "{{ dnetworks }}" - when: docker_py_version is version('1.10.0', '>=') and docker_api_version is version('1.25', '>=') + when: docker_api_version is version('1.25', '>=') # FIXME: find out API version! - fail: msg="Too old docker / docker-py version to run docker_network tests!" - when: not(docker_py_version is version('1.10.0', '>=') and docker_api_version is version('1.25', '>=')) and (ansible_distribution != 'CentOS' or ansible_distribution_major_version|int > 6) + when: not(docker_api_version is version('1.25', '>=')) and (ansible_distribution != 'CentOS' or ansible_distribution_major_version|int > 6) diff --git a/tests/integration/targets/docker_network/tasks/tests/ipam.yml b/tests/integration/targets/docker_network/tasks/tests/ipam.yml index 7491150b6..d3fccdbd0 100644 --- a/tests/integration/targets/docker_network/tasks/tests/ipam.yml +++ b/tests/integration/targets/docker_network/tasks/tests/ipam.yml @@ -303,10 +303,3 @@ - network_1 is changed - network_2 is not changed - network_3 is changed - when: docker_py_version is version('2.0.0', '>=') -- assert: - that: - - network_1 is failed - - "('version is ' ~ docker_py_version ~ ' ') in network_1.msg" - - "'Minimum version required is 2.0.0 ' in network_1.msg" - when: docker_py_version is version('2.0.0', '<') diff --git a/tests/integration/targets/docker_network/tasks/tests/options.yml b/tests/integration/targets/docker_network/tasks/tests/options.yml index 7723cabfa..f8b74b5f5 100644 --- a/tests/integration/targets/docker_network/tasks/tests/options.yml +++ b/tests/integration/targets/docker_network/tasks/tests/options.yml @@ -144,9 +144,6 @@ state: absent force: yes - # Requirements for docker_swarm - when: docker_py_version is version('2.6.0', '>=') - #################################################################### ## attachable ###################################################### #################################################################### @@ -183,13 +180,6 @@ - attachable_1 is changed - attachable_2 is not changed - attachable_3 is changed - when: docker_py_version is version('2.0.0', '>=') -- assert: - that: - - attachable_1 is failed - - "('version is ' ~ docker_py_version ~ ' ') in attachable_1.msg" - - "'Minimum version required is 2.0.0 ' in attachable_1.msg" - when: docker_py_version is version('2.0.0', '<') #################################################################### ## labels ########################################################## diff --git a/tests/integration/targets/docker_network_info/tasks/main.yml b/tests/integration/targets/docker_network_info/tasks/main.yml index 4d4ff698f..34f46559a 100644 --- a/tests/integration/targets/docker_network_info/tasks/main.yml +++ b/tests/integration/targets/docker_network_info/tasks/main.yml @@ -70,7 +70,7 @@ - "'is too new. Maximum supported API version is' in docker_inspect.stderr" when: docker_inspect is failed - when: docker_py_version is version('1.8.0', '>=') and docker_api_version is version('1.25', '>=') + when: docker_api_version is version('1.25', '>=') - fail: msg="Too old docker / docker-py version to run docker_network_info tests!" - when: not(docker_py_version is version('1.8.0', '>=') and docker_api_version is version('1.25', '>=')) and (ansible_distribution != 'CentOS' or ansible_distribution_major_version|int > 6) + when: not(docker_api_version is version('1.25', '>=')) and (ansible_distribution != 'CentOS' or ansible_distribution_major_version|int > 6) diff --git a/tests/integration/targets/docker_volume/tasks/main.yml b/tests/integration/targets/docker_volume/tasks/main.yml index 906d16156..c796fb574 100644 --- a/tests/integration/targets/docker_volume/tasks/main.yml +++ b/tests/integration/targets/docker_volume/tasks/main.yml @@ -24,7 +24,7 @@ state: absent with_items: "{{ vnames }}" - when: docker_py_version is version('1.10.0', '>=') and docker_api_version is version('1.25', '>=') # FIXME: find out API version! + when: docker_api_version is version('1.25', '>=') - fail: msg="Too old docker / docker-py version to run docker_volume tests!" - when: not(docker_py_version is version('1.10.0', '>=') and docker_api_version is version('1.25', '>=')) and (ansible_distribution != 'CentOS' or ansible_distribution_major_version|int > 6) + when: not(docker_api_version is version('1.25', '>=')) and (ansible_distribution != 'CentOS' or ansible_distribution_major_version|int > 6) diff --git a/tests/integration/targets/docker_volume_info/tasks/main.yml b/tests/integration/targets/docker_volume_info/tasks/main.yml index 1ee580f89..e7007b72c 100644 --- a/tests/integration/targets/docker_volume_info/tasks/main.yml +++ b/tests/integration/targets/docker_volume_info/tasks/main.yml @@ -67,8 +67,7 @@ - "'is too new. Maximum supported API version is' in docker_volume_inspect.stderr" when: docker_volume_inspect is failed - # Requirements for docker_volume - when: docker_py_version is version('1.10.0', '>=') and docker_api_version is version('1.25', '>=') + when: docker_api_version is version('1.25', '>=') - fail: msg="Too old docker / docker-py version to run docker_volume_info tests!" - when: not(docker_py_version is version('1.10.0', '>=') and docker_api_version is version('1.25', '>=')) and (ansible_distribution != 'CentOS' or ansible_distribution_major_version|int > 6) + when: not(docker_api_version is version('1.25', '>=')) and (ansible_distribution != 'CentOS' or ansible_distribution_major_version|int > 6) From d28db0aab36c2f50ca488297eb67a9d22bb253d6 Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Mon, 20 Jun 2022 13:50:41 +0200 Subject: [PATCH 07/14] Adjust docker_container_exec. --- plugins/modules/docker_container_exec.py | 72 ++++++++----------- .../docker_container_exec/tasks/main.yml | 46 ++++++------ 2 files changed, 52 insertions(+), 66 deletions(-) diff --git a/plugins/modules/docker_container_exec.py b/plugins/modules/docker_container_exec.py index cc0eecc8a..749c8fa2e 100644 --- a/plugins/modules/docker_container_exec.py +++ b/plugins/modules/docker_container_exec.py @@ -82,15 +82,13 @@ version_added: 2.1.0 extends_documentation_fragment: - - community.docker.docker - - community.docker.docker.docker_py_1_documentation + - community.docker.docker.api_documentation notes: - Does not support C(check_mode). author: - "Felix Fontein (@felixfontein)" requirements: - - "L(Docker SDK for Python,https://docker-py.readthedocs.io/en/stable/) >= 1.8.0" - "Docker API >= 1.25" ''' @@ -154,7 +152,7 @@ from ansible.module_utils.compat import selectors from ansible.module_utils.six import string_types -from ansible_collections.community.docker.plugins.module_utils.common import ( +from ansible_collections.community.docker.plugins.module_utils.common_api import ( AnsibleDockerClient, RequestException, ) @@ -168,11 +166,12 @@ DockerSocketHandlerModule, ) -try: - from docker.errors import DockerException, APIError, NotFound -except Exception: - # missing Docker SDK for Python handled in ansible.module_utils.docker.common - pass +from ansible_collections.community.docker.plugins.module_utils._api.errors import ( + APIError, + DockerException, + NotFound, +) +from ansible_collections.community.docker.plugins.module_utils._api.utils.utils import format_environment def main(): @@ -191,8 +190,7 @@ def main(): ) option_minimal_versions = dict( - chdir=dict(docker_py_version='3.0.0', docker_api_version='1.35'), - env=dict(docker_py_version='2.3.0'), + chdir=dict(docker_api_version='1.35'), ) client = AnsibleDockerClient( @@ -231,34 +229,31 @@ def main(): stdin += '\n' try: - kwargs = {} - if chdir is not None: - kwargs['workdir'] = chdir - if env is not None: - kwargs['environment'] = env - exec_data = client.exec_create( - container, - argv, - stdout=True, - stderr=True, - stdin=bool(stdin), - user=user or '', - **kwargs - ) + data = { + 'Container': container, + 'User': user or '', + 'Privileged': False, + 'Tty': False, + 'AttachStdin': bool(stdin), + 'AttachStdout': True, + 'AttachStderr': True, + 'Cmd': argv, + 'Env': format_environment(env) if env is not None else None, + } + exec_data = client.post_json_to_json('/containers/{0}/exec', container, data=data) exec_id = exec_data['Id'] + data = { + 'Tty': tty, + 'Detach': detach, + } if detach: - client.exec_start(exec_id, tty=tty, detach=True) + client.post_json_to_text('/exec/{0}/start', exec_id, data=data) client.module.exit_json(changed=True, exec_id=exec_id) else: if stdin and not detach: - exec_socket = client.exec_start( - exec_id, - tty=tty, - detach=False, - socket=True, - ) + exec_socket = client.post_json_to_stream_socket('/exec/{0}/start', exec_id, data=data) try: with DockerSocketHandlerModule(exec_socket, client.module, selectors) as exec_socket_handler: if stdin: @@ -268,16 +263,9 @@ def main(): finally: exec_socket.close() else: - stdout, stderr = client.exec_start( - exec_id, - tty=tty, - detach=False, - stream=False, - socket=False, - demux=True, - ) - - result = client.exec_inspect(exec_id) + stdout, stderr = client.post_json_to_stream('/exec/{0}/start', exec_id, data=data, stream=False, tty=tty, demux=True) + + result = client.get_json('/exec/{0}/json', exec_id) stdout = to_text(stdout or b'') stderr = to_text(stderr or b'') diff --git a/tests/integration/targets/docker_container_exec/tasks/main.yml b/tests/integration/targets/docker_container_exec/tasks/main.yml index c72b9adaf..b3d57745c 100644 --- a/tests/integration/targets/docker_container_exec/tasks/main.yml +++ b/tests/integration/targets/docker_container_exec/tasks/main.yml @@ -190,28 +190,26 @@ - "'stderr' not in result" - result.exec_id is string - - when: docker_py_version is version('2.3.0', '>=') - block: - - name: Execute in a present container (environment variable) - docker_container_exec: - container: "{{ cname }}" - argv: - - /bin/sh - - '-c' - - 'echo "$FOO" ; echo $FOO > /dev/stderr' - env: - FOO: |- - bar - baz - register: result - - - assert: - that: - - result.rc == 0 - - result.stdout == 'bar\nbaz' - - result.stdout_lines == ['bar', 'baz'] - - result.stderr == 'bar baz' - - result.stderr_lines == ['bar baz'] + - name: Execute in a present container (environment variable) + docker_container_exec: + container: "{{ cname }}" + argv: + - /bin/sh + - '-c' + - 'echo "$FOO" ; echo $FOO > /dev/stderr' + env: + FOO: |- + bar + baz + register: result + + - assert: + that: + - result.rc == 0 + - result.stdout == 'bar\nbaz' + - result.stdout_lines == ['bar', 'baz'] + - result.stderr == 'bar baz' + - result.stderr_lines == ['bar baz'] always: - name: Cleanup @@ -220,7 +218,7 @@ state: absent force_kill: yes - when: docker_py_version is version('1.8.0', '>=') and docker_api_version is version('1.25', '>=') + when: docker_api_version is version('1.25', '>=') - fail: msg="Too old docker / docker-py version to run docker_container_exec tests!" - when: not(docker_py_version is version('1.8.0', '>=') and docker_api_version is version('1.25', '>=')) and (ansible_distribution != 'CentOS' or ansible_distribution_major_version|int > 6) + when: not(docker_api_version is version('1.25', '>=')) and (ansible_distribution != 'CentOS' or ansible_distribution_major_version|int > 6) From 3be80bde72432cfbd70ad9227f17b23b9de10a9f Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Fri, 24 Jun 2022 08:15:59 +0200 Subject: [PATCH 08/14] Adjust docker_image. --- plugins/modules/docker_image.py | 253 ++++++++++-------- .../targets/docker_image/tasks/test.yml | 4 +- .../docker_image/tasks/tests/options.yml | 14 - 3 files changed, 148 insertions(+), 123 deletions(-) diff --git a/plugins/modules/docker_image.py b/plugins/modules/docker_image.py index a5625eb13..43ab27c15 100644 --- a/plugins/modules/docker_image.py +++ b/plugins/modules/docker_image.py @@ -207,12 +207,10 @@ default: latest extends_documentation_fragment: -- community.docker.docker -- community.docker.docker.docker_py_1_documentation + - community.docker.docker.api_documentation requirements: - - "L(Docker SDK for Python,https://docker-py.readthedocs.io/en/stable/) >= 1.8.0" - "Docker API >= 1.25" author: @@ -324,13 +322,15 @@ ''' import errno +import json import os import traceback -from ansible_collections.community.docker.plugins.module_utils.common import ( +from ansible.module_utils.common.text.converters import to_native + +from ansible_collections.community.docker.plugins.module_utils.common_api import ( AnsibleDockerClient, RequestException, - docker_version, ) from ansible_collections.community.docker.plugins.module_utils.util import ( clean_dict_booleans_for_docker_api, @@ -338,21 +338,25 @@ is_image_name_id, is_valid_tag, ) -from ansible.module_utils.common.text.converters import to_native - from ansible_collections.community.docker.plugins.module_utils.version import LooseVersion -if docker_version is not None: - try: - if LooseVersion(docker_version) >= LooseVersion('2.0.0'): - from docker.auth import resolve_repository_name - else: - from docker.auth.auth import resolve_repository_name - from docker.utils.utils import parse_repository_tag - from docker.errors import DockerException, NotFound - except ImportError: - # missing Docker SDK for Python handled in module_utils.docker.common - pass +from ansible_collections.community.docker.plugins.module_utils._api.auth import ( + get_config_header, + resolve_repository_name, +) +from ansible_collections.community.docker.plugins.module_utils._api.constants import ( + DEFAULT_DATA_CHUNK_SIZE, + CONTAINER_LIMITS_KEYS, +) +from ansible_collections.community.docker.plugins.module_utils._api.errors import DockerException, NotFound +from ansible_collections.community.docker.plugins.module_utils._api.utils.build import ( + process_dockerfile, + tar, +) +from ansible_collections.community.docker.plugins.module_utils._api.utils.utils import ( + format_extra_hosts, + parse_repository_tag, +) class ImageManager(DockerBaseClass): @@ -501,7 +505,7 @@ def absent(self): if image: if not self.check_mode: try: - self.client.remove_image(name, force=self.force_absent) + self.client.delete_json('/images/{0}', name, params={'force': self.force_absent}) except NotFound: # If the image vanished while we were trying to remove it, don't fail pass @@ -539,18 +543,18 @@ def archive_image(self, name, tag): if not self.check_mode: self.log("Getting archive of image %s" % image_name) try: - saved_image = self.client.get_image(image_name) + saved_image = self.client._stream_raw_result( + self.client._get(self.client._url('/images/{0}/get', image_name), stream=True), + DEFAULT_DATA_CHUNK_SIZE, + False, + ) except Exception as exc: self.fail("Error getting image %s - %s" % (image_name, to_native(exc))) try: with open(self.archive_path, 'wb') as fd: - if self.client.docker_py_version >= LooseVersion('3.0.0'): - for chunk in saved_image: - fd.write(chunk) - else: - for chunk in saved_image.stream(2048, decode_content=False): - fd.write(chunk) + for chunk in saved_image: + fd.write(chunk) except Exception as exc: self.fail("Error writing image archive %s - %s" % (self.archive_path, to_native(exc))) @@ -583,7 +587,24 @@ def push_image(self, name, tag=None): status = None try: changed = False - for line in self.client.push(repository, tag=tag, stream=True, decode=True): + + push_repository, push_tag = repository, tag + if not push_tag: + push_repository, push_tag = parse_repository_tag(push_repository) + push_registry, dummy = resolve_repository_name(push_repository) + headers = {} + header = get_config_header(self.client, push_registry) + if header: + headers['X-Registry-Auth'] = header + response = self.client._post_json( + self.client._url("/images/{0}/push", push_repository), + data=None, + headers=headers, + stream=True, + params={'tag': push_tag}, + ) + self.client._raise_for_status(response) + for line in self.client._stream_helper(response, decode=True): self.log(line, pretty_print=True) if line.get('errorDetail'): raise Exception(line['errorDetail']['message']) @@ -635,8 +656,14 @@ def tag_image(self, name, tag, repository, push=False): try: # Finding the image does not always work, especially running a localhost registry. In those # cases, if we don't set force=True, it errors. - tag_status = self.client.tag(image_name, repo, tag=repo_tag, force=True) - if not tag_status: + params = { + 'tag': repo_tag, + 'repo': repo, + 'force': True, + } + res = self.client._post(self.client._url('/images/{0}/tag', image_name), params=params) + self.client._raise_for_status(res) + if res.status_code != 201: raise Exception("Tag operation failed.") except Exception as exc: self.fail("Error: failed to tag image - %s" % to_native(exc)) @@ -664,47 +691,88 @@ def build_image(self): :return: image dict ''' - params = dict( - path=self.build_path, - tag=self.name, - rm=self.rm, - nocache=self.nocache, - timeout=self.http_timeout, - pull=self.pull, - forcerm=self.rm, - dockerfile=self.dockerfile, - decode=True, - ) - if self.client.docker_py_version < LooseVersion('3.0.0'): - params['stream'] = True - - if self.tag: - params['tag'] = "%s:%s" % (self.name, self.tag) - if self.container_limits: - params['container_limits'] = self.container_limits + remote = context = None + headers = {} + buildargs = {} if self.buildargs: for key, value in self.buildargs.items(): self.buildargs[key] = to_native(value) - params['buildargs'] = self.buildargs - if self.cache_from: - params['cache_from'] = self.cache_from - if self.network: - params['network_mode'] = self.network - if self.extra_hosts: - params['extra_hosts'] = self.extra_hosts + + container_limits = self.container_limits or {} + for key in container_limits.keys(): + if key not in CONTAINER_LIMITS_KEYS: + raise DockerException('Invalid container_limits key {key}'.format(key=key)) + + dockerfile = self.dockerfile + if self.build_path.startswith(('http://', 'https://', 'git://', 'github.com/', 'git@')): + remote = self.build_path + elif not os.path.isdir(self.build_path): + raise TypeError("You must specify a directory to build in path") + else: + dockerignore = os.path.join(self.build_path, '.dockerignore') + exclude = None + if os.path.exists(dockerignore): + with open(dockerignore) as f: + exclude = list(filter( + lambda x: x != '' and x[0] != '#', + [line.strip() for line in f.read().splitlines()] + )) + dockerfile = process_dockerfile(dockerfile, self.build_path) + context = tar(self.build_path, exclude=exclude, dockerfile=dockerfile, gzip=False) + + params = { + 't': "%s:%s" % (self.name, self.tag) if self.tag else self.name, + 'remote': remote, + 'q': False, + 'nocache': self.nocache, + 'rm': self.rm, + 'forcerm': self.rm, + 'pull': self.pull, + 'dockerfile': dockerfile, + } + params.update(container_limits) + if self.use_config_proxy: - params['use_config_proxy'] = self.use_config_proxy - # Due to a bug in Docker SDK for Python, it will crash if - # use_config_proxy is True and buildargs is None - if 'buildargs' not in params: - params['buildargs'] = {} + proxy_args = self.client._proxy_configs.get_environment() + for k, v in proxy_args.items(): + buildargs.setdefault(k, v) + if buildargs: + params.update({'buildargs': json.dumps(buildargs)}) + + if self.cache_from: + params.update({'cachefrom': json.dumps(self.cache_from)}) + if self.target: - params['target'] = self.target + params.update({'target': self.target}) + + if self.network: + params.update({'networkmode': self.network}) + + if self.extra_hosts is not None: + params.update({'extrahosts': format_extra_hosts(self.extra_hosts)}) + if self.build_platform is not None: params['platform'] = self.build_platform + if context is not None: + headers['Content-Type'] = 'application/tar' + + self.client._set_auth_headers(headers) + + response = self.client._post( + self.client._url('/build'), + data=context, + params=params, + headers=headers, + stream=True, + timeout=self.http_timeout, + ) + + if context is not None: + context.close() + build_output = [] - for line in self.client.build(**params): + for line in self.client._stream_helper(response, decode=True): # line = json.loads(line) self.log(line, pretty_print=True) self._extract_output_line(line, build_output) @@ -722,8 +790,10 @@ def build_image(self): self.fail("Error building %s - message: %s, logs: %s" % ( self.name, line.get('error'), build_output)) - return {"stdout": "\n".join(build_output), - "image": self.client.find_image(name=self.name, tag=self.tag)} + return { + "stdout": "\n".join(build_output), + "image": self.client.find_image(name=self.name, tag=self.tag), + } def load_image(self): ''' @@ -738,34 +808,20 @@ def load_image(self): self.log("Opening image %s" % self.load_path) with open(self.load_path, 'rb') as image_tar: self.log("Loading image from %s" % self.load_path) - output = self.client.load_image(image_tar) - if output is not None: - # Old versions of Docker SDK of Python (before version 2.5.0) do not return anything. - # (See https://github.com/docker/docker-py/commit/7139e2d8f1ea82340417add02090bfaf7794f159) - # Note that before that commit, something else than None was returned, but that was also - # only introduced in a commit that first appeared in 2.5.0 (see - # https://github.com/docker/docker-py/commit/9e793806ff79559c3bc591d8c52a3bbe3cdb7350). - # So the above check works for every released version of Docker SDK for Python. + res = self.client._post(self.client._url("/images/load"), data=image_tar, stream=True) + if LooseVersion(self.client.api_version) >= LooseVersion('1.23'): has_output = True - for line in output: + for line in self.client._stream_helper(res, decode=True): self.log(line, pretty_print=True) self._extract_output_line(line, load_output) else: - if LooseVersion(docker_version) < LooseVersion('2.5.0'): - self.client.module.warn( - 'The installed version of the Docker SDK for Python does not return the loading results' - ' from the Docker daemon. Therefore, we cannot verify whether the expected image was' - ' loaded, whether multiple images where loaded, or whether the load actually succeeded.' - ' If you are not stuck with Python 2.6, *please* upgrade to a version newer than 2.5.0' - ' (2.5.0 was released in August 2017).' - ) - else: - self.client.module.warn( - 'The API version of your Docker daemon is < 1.23, which does not return the image' - ' loading result from the Docker daemon. Therefore, we cannot verify whether the' - ' expected image was loaded, whether multiple images where loaded, or whether the load' - ' actually succeeded. You should consider upgrading your Docker daemon.' - ) + self.client._raise_for_status(res) + self.client.module.warn( + 'The API version of your Docker daemon is < 1.23, which does not return the image' + ' loading result from the Docker daemon. Therefore, we cannot verify whether the' + ' expected image was loaded, whether multiple images where loaded, or whether the load' + ' actually succeeded. You should consider upgrading your Docker daemon.' + ) except EnvironmentError as exc: if exc.errno == errno.ENOENT: self.client.fail("Error opening image %s - %s" % (self.load_path, to_native(exc))) @@ -857,18 +913,6 @@ def main(): ('source', 'load', ['load_path']), ] - def detect_build_cache_from(client): - return client.module.params['build'] and client.module.params['build'].get('cache_from') is not None - - def detect_build_network(client): - return client.module.params['build'] and client.module.params['build'].get('network') is not None - - def detect_build_target(client): - return client.module.params['build'] and client.module.params['build'].get('target') is not None - - def detect_use_config_proxy(client): - return client.module.params['build'] and client.module.params['build'].get('use_config_proxy') is not None - def detect_etc_hosts(client): return client.module.params['build'] and bool(client.module.params['build'].get('etc_hosts')) @@ -879,19 +923,14 @@ def detect_pull_platform(client): return client.module.params['pull'] and client.module.params['pull'].get('platform') is not None option_minimal_versions = dict() - option_minimal_versions["build.cache_from"] = dict(docker_py_version='2.1.0', detect_usage=detect_build_cache_from) - option_minimal_versions["build.network"] = dict(docker_py_version='2.4.0', detect_usage=detect_build_network) - option_minimal_versions["build.target"] = dict(docker_py_version='2.4.0', detect_usage=detect_build_target) - option_minimal_versions["build.use_config_proxy"] = dict(docker_py_version='3.7.0', detect_usage=detect_use_config_proxy) - option_minimal_versions["build.etc_hosts"] = dict(docker_py_version='2.6.0', docker_api_version='1.27', detect_usage=detect_etc_hosts) - option_minimal_versions["build.platform"] = dict(docker_py_version='3.0.0', docker_api_version='1.32', detect_usage=detect_build_platform) - option_minimal_versions["pull.platform"] = dict(docker_py_version='3.0.0', docker_api_version='1.32', detect_usage=detect_pull_platform) + option_minimal_versions["build.etc_hosts"] = dict(docker_api_version='1.27', detect_usage=detect_etc_hosts) + option_minimal_versions["build.platform"] = dict(docker_api_version='1.32', detect_usage=detect_build_platform) + option_minimal_versions["pull.platform"] = dict(docker_api_version='1.32', detect_usage=detect_pull_platform) client = AnsibleDockerClient( argument_spec=argument_spec, required_if=required_if, supports_check_mode=True, - min_docker_version='1.8.0', option_minimal_versions=option_minimal_versions, ) diff --git a/tests/integration/targets/docker_image/tasks/test.yml b/tests/integration/targets/docker_image/tasks/test.yml index 2962769fe..a432f5d72 100644 --- a/tests/integration/targets/docker_image/tasks/test.yml +++ b/tests/integration/targets/docker_image/tasks/test.yml @@ -43,7 +43,7 @@ force_kill: yes with_items: "{{ cnames }}" - when: docker_py_version is version('1.8.0', '>=') and docker_api_version is version('1.25', '>=') + when: docker_api_version is version('1.25', '>=') - fail: msg="Too old docker / docker-py version to run docker_image tests!" - when: not(docker_py_version is version('1.8.0', '>=') and docker_api_version is version('1.25', '>=')) and (ansible_distribution != 'CentOS' or ansible_distribution_major_version|int > 6) + when: not(docker_api_version is version('1.25', '>=')) and (ansible_distribution != 'CentOS' or ansible_distribution_major_version|int > 6) diff --git a/tests/integration/targets/docker_image/tasks/tests/options.yml b/tests/integration/targets/docker_image/tasks/tests/options.yml index 267045f7a..c873b4e6d 100644 --- a/tests/integration/targets/docker_image/tasks/tests/options.yml +++ b/tests/integration/targets/docker_image/tasks/tests/options.yml @@ -56,13 +56,6 @@ that: - buildargs_1 is changed - buildargs_2 is not failed and buildargs_2 is not changed - when: docker_py_version is version('1.6.0', '>=') - -- assert: - that: - - buildargs_1 is failed - - buildargs_2 is failed - when: docker_py_version is version('1.6.0', '<') #################################################################### ## build.container_limits ########################################## @@ -177,13 +170,6 @@ that: - platform_1 is changed - platform_2 is not failed and platform_2 is not changed - when: docker_py_version is version('3.0.0', '>=') - -- assert: - that: - - platform_1 is failed - - platform_2 is failed - when: docker_py_version is version('3.0.0', '<') #################################################################### ## force ########################################################### From 300b2b4edfe5d1e04b08b59c39b694de35619cc7 Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Fri, 24 Jun 2022 08:46:42 +0200 Subject: [PATCH 09/14] Adjust docker_image_load module. --- plugins/modules/docker_image_load.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/plugins/modules/docker_image_load.py b/plugins/modules/docker_image_load.py index 3ace84d3c..077bbe910 100644 --- a/plugins/modules/docker_image_load.py +++ b/plugins/modules/docker_image_load.py @@ -28,14 +28,12 @@ required: true extends_documentation_fragment: -- community.docker.docker -- community.docker.docker.docker_py_2_documentation + - community.docker.docker.api_documentation notes: - Does not support C(check_mode). requirements: - - "L(Docker SDK for Python,https://docker-py.readthedocs.io/en/stable/) >= 2.5.0" - "Docker API >= 1.25" author: @@ -75,7 +73,7 @@ from ansible.module_utils.common.text.converters import to_native -from ansible_collections.community.docker.plugins.module_utils.common import ( +from ansible_collections.community.docker.plugins.module_utils.common_api import ( AnsibleDockerClient, RequestException, ) @@ -84,11 +82,7 @@ is_image_name_id, ) -try: - from docker.errors import DockerException -except ImportError: - # missing Docker SDK for Python handled in module_utils.docker.common - pass +from ansible_collections.community.docker.plugins.module_utils._api.errors import DockerException class ImageManager(DockerBaseClass): @@ -125,7 +119,8 @@ def load_images(self): self.log("Opening image {0}".format(self.path)) with open(self.path, 'rb') as image_tar: self.log("Loading images from {0}".format(self.path)) - for line in self.client.load_image(image_tar): + res = self.client._post(self.client._url("/images/load"), data=image_tar, stream=True) + for line in self.client._stream_helper(res, decode=True): self.log(line, pretty_print=True) self._extract_output_line(line, load_output) except EnvironmentError as exc: @@ -168,7 +163,6 @@ def main(): path=dict(type='path', required=True), ), supports_check_mode=False, - min_docker_version='2.5.0', ) try: From a67a385e614313a7f6842464807affbb4b923695 Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Fri, 24 Jun 2022 21:14:52 +0200 Subject: [PATCH 10/14] Adjust docker_prune module. --- plugins/modules/docker_image.py | 1 - plugins/modules/docker_prune.py | 45 +++++++------------ .../targets/docker_prune/tasks/main.yml | 9 ++-- 3 files changed, 19 insertions(+), 36 deletions(-) diff --git a/plugins/modules/docker_image.py b/plugins/modules/docker_image.py index 43ab27c15..a208d307b 100644 --- a/plugins/modules/docker_image.py +++ b/plugins/modules/docker_image.py @@ -117,7 +117,6 @@ - If set to C(yes) and a proxy configuration is specified in the docker client configuration (by default C($HOME/.docker/config.json)), the corresponding environment variables will be set in the container being built. - - Needs Docker SDK for Python >= 3.7.0. type: bool target: description: diff --git a/plugins/modules/docker_prune.py b/plugins/modules/docker_prune.py index 9225fb814..9fb82c730 100644 --- a/plugins/modules/docker_prune.py +++ b/plugins/modules/docker_prune.py @@ -68,20 +68,17 @@ builder_cache: description: - Whether to prune the builder cache. - - Requires version 3.3.0 of the Docker SDK for Python or newer. type: bool default: no extends_documentation_fragment: -- community.docker.docker -- community.docker.docker.docker_py_2_documentation + - community.docker.docker.api_documentation author: - "Felix Fontein (@felixfontein)" requirements: - - "L(Docker SDK for Python,https://docker-py.readthedocs.io/en/stable/) >= 2.1.0" - "Docker API >= 1.25" ''' @@ -180,24 +177,15 @@ from ansible.module_utils.common.text.converters import to_native -try: - from docker.errors import DockerException -except ImportError: - # missing Docker SDK for Python handled in ansible.module_utils.docker.common - pass - -from ansible_collections.community.docker.plugins.module_utils.version import LooseVersion - -from ansible_collections.community.docker.plugins.module_utils.common import ( +from ansible_collections.community.docker.plugins.module_utils.common_api import ( AnsibleDockerClient, RequestException, ) -try: - from ansible_collections.community.docker.plugins.module_utils.common import docker_version, clean_dict_booleans_for_docker_api -except Exception as dummy: - # missing Docker SDK for Python handled in ansible.module_utils.docker.common - pass +from ansible_collections.community.docker.plugins.module_utils.util import clean_dict_booleans_for_docker_api + +from ansible_collections.community.docker.plugins.module_utils._api.errors import DockerException +from ansible_collections.community.docker.plugins.module_utils._api.utils.utils import convert_filters def main(): @@ -216,43 +204,40 @@ def main(): client = AnsibleDockerClient( argument_spec=argument_spec, # supports_check_mode=True, - min_docker_version='2.1.0', ) - # Version checks - cache_min_version = '3.3.0' - if client.module.params['builder_cache'] and client.docker_py_version < LooseVersion(cache_min_version): - msg = "Error: Docker SDK for Python's version is %s. Minimum version required for builds option is %s. Use `pip install --upgrade docker` to upgrade." - client.fail(msg % (docker_version, cache_min_version)) - try: result = dict() if client.module.params['containers']: filters = clean_dict_booleans_for_docker_api(client.module.params.get('containers_filters')) - res = client.prune_containers(filters=filters) + params = {'filters': convert_filters(filters)} + res = client.post_to_json('/containers/prune', params=params) result['containers'] = res.get('ContainersDeleted') or [] result['containers_space_reclaimed'] = res['SpaceReclaimed'] if client.module.params['images']: filters = clean_dict_booleans_for_docker_api(client.module.params.get('images_filters')) - res = client.prune_images(filters=filters) + params = {'filters': convert_filters(filters)} + res = client.post_to_json('/images/prune', params=params) result['images'] = res.get('ImagesDeleted') or [] result['images_space_reclaimed'] = res['SpaceReclaimed'] if client.module.params['networks']: filters = clean_dict_booleans_for_docker_api(client.module.params.get('networks_filters')) - res = client.prune_networks(filters=filters) + params = {'filters': convert_filters(filters)} + res = client.post_to_json('/networks/prune', params=params) result['networks'] = res.get('NetworksDeleted') or [] if client.module.params['volumes']: filters = clean_dict_booleans_for_docker_api(client.module.params.get('volumes_filters')) - res = client.prune_volumes(filters=filters) + params = {'filters': convert_filters(filters)} + res = client.post_to_json('/volumes/prune', params=params) result['volumes'] = res.get('VolumesDeleted') or [] result['volumes_space_reclaimed'] = res['SpaceReclaimed'] if client.module.params['builder_cache']: - res = client.prune_builds() + res = client.post_to_json('/build/prune') result['builder_cache_space_reclaimed'] = res['SpaceReclaimed'] client.module.exit_json(**result) diff --git a/tests/integration/targets/docker_prune/tasks/main.yml b/tests/integration/targets/docker_prune/tasks/main.yml index 16c4aa058..2a8ab4509 100644 --- a/tests/integration/targets/docker_prune/tasks/main.yml +++ b/tests/integration/targets/docker_prune/tasks/main.yml @@ -32,7 +32,7 @@ images: yes networks: yes volumes: yes - builder_cache: "{{ docker_py_version is version('3.3.0', '>=') }}" + builder_cache: yes register: result # Analyze result @@ -50,8 +50,7 @@ - volume.volume.Name in result.volumes - "'volumes_space_reclaimed' in result" # builder_cache - - "'builder_cache_space_reclaimed' in result or docker_py_version is version('3.3.0', '<')" - - "'builder_cache_space_reclaimed' not in result or docker_py_version is version('3.3.0', '>=')" + - "'builder_cache_space_reclaimed' in result" # Test with filters - docker_prune: @@ -62,7 +61,7 @@ - debug: var=result - when: docker_py_version is version('2.1.0', '>=') and docker_api_version is version('1.25', '>=') + when: docker_api_version is version('1.25', '>=') - fail: msg="Too old docker / docker-py version to run docker_prune tests!" - when: not(docker_py_version is version('2.1.0', '>=') and docker_api_version is version('1.25', '>=')) and (ansible_distribution != 'CentOS' or ansible_distribution_major_version|int > 6) + when: not(docker_api_version is version('1.25', '>=')) and (ansible_distribution != 'CentOS' or ansible_distribution_major_version|int > 6) From d22960c547c8617c6a7df6549d25013a192b82f2 Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Fri, 24 Jun 2022 21:56:54 +0200 Subject: [PATCH 11/14] Adjust docker_login. --- plugins/modules/docker_login.py | 118 +++++++----------- .../targets/docker_login/tasks/test.yml | 4 +- 2 files changed, 44 insertions(+), 78 deletions(-) diff --git a/plugins/modules/docker_login.py b/plugins/modules/docker_login.py index 33a3b8438..93b50eda8 100644 --- a/plugins/modules/docker_login.py +++ b/plugins/modules/docker_login.py @@ -65,13 +65,9 @@ choices: ['present', 'absent'] extends_documentation_fragment: -- community.docker.docker -- community.docker.docker.docker_py_1_documentation + - community.docker.docker.api_documentation requirements: - - "L(Docker SDK for Python,https://docker-py.readthedocs.io/en/stable/) >= 1.8.0" - - "Python bindings for docker credentials store API >= 0.2.1 - (use L(docker-pycreds,https://pypi.org/project/docker-pycreds/) when using Docker SDK for Python < 4.0.0)" - "Docker API >= 1.25" author: - Olaf Kilian (@olsaki) @@ -106,7 +102,7 @@ RETURN = ''' login_results: description: Results from the login. - returned: when state='present' + returned: when I(state=present) type: dict sample: { "serveraddress": "localhost:5000", @@ -117,28 +113,11 @@ import base64 import json import os -import re import traceback from ansible.module_utils.common.text.converters import to_bytes, to_text, to_native -try: - from docker.errors import DockerException - from docker import auth - - # Earlier versions of docker/docker-py put decode_auth - # in docker.auth.auth instead of docker.auth - if hasattr(auth, 'decode_auth'): - from docker.auth import decode_auth - else: - from docker.auth.auth import decode_auth - -except ImportError: - # missing Docker SDK for Python handled in ansible.module_utils.docker.common - pass - -from ansible_collections.community.docker.plugins.module_utils.common import ( - HAS_DOCKER_PY, +from ansible_collections.community.docker.plugins.module_utils.common_api import ( AnsibleDockerClient, RequestException, ) @@ -147,32 +126,11 @@ DockerBaseClass, ) -NEEDS_DOCKER_PYCREDS = False - -# Early versions of docker/docker-py rely on docker-pycreds for -# the credential store api. -if HAS_DOCKER_PY: - try: - from docker.credentials.errors import StoreError, CredentialsNotFound - from docker.credentials import Store - except ImportError: - try: - from dockerpycreds.errors import StoreError, CredentialsNotFound - from dockerpycreds.store import Store - except ImportError as exc: - HAS_DOCKER_ERROR = str(exc) - NEEDS_DOCKER_PYCREDS = True - - -if NEEDS_DOCKER_PYCREDS: - # docker-pycreds missing, so we need to create some place holder classes - # to allow instantiation. - - class StoreError(Exception): - pass - - class CredentialsNotFound(Exception): - pass +from ansible_collections.community.docker.plugins.module_utils._api import auth +from ansible_collections.community.docker.plugins.module_utils._api.auth import decode_auth +from ansible_collections.community.docker.plugins.module_utils._api.credentials.errors import CredentialsNotFound +from ansible_collections.community.docker.plugins.module_utils._api.credentials.store import Store +from ansible_collections.community.docker.plugins.module_utils._api.errors import DockerException class DockerFileStore(object): @@ -305,6 +263,35 @@ def run(self): def fail(self, msg): self.client.fail(msg) + def _login(self, reauth): + if self.config_path and os.path.exists(self.config_path): + self.client._auth_configs = auth.load_config( + self.config_path, credstore_env=self.client.credstore_env + ) + elif not self.client._auth_configs or self.client._auth_configs.is_empty: + self.client._auth_configs = auth.load_config( + credstore_env=self.client.credstore_env + ) + + authcfg = self.client._auth_configs.resolve_authconfig(self.registry_url) + # If we found an existing auth config for this registry and username + # combination, we can return it immediately unless reauth is requested. + if authcfg and authcfg.get('username', None) == self.username \ + and not reauth: + return authcfg + + req_data = { + 'username': self.username, + 'password': self.password, + 'email': None, + 'serveraddress': self.registry_url, + } + + response = self.client._post_json(self.client._url('/auth'), data=req_data) + if response.status_code == 200: + self.client._auth_configs.add_auth(self.registry_url or auth.INDEX_NAME, req_data) + return self.client._result(response, json=True) + def login(self): ''' Log into the registry with provided username/password. On success update the config @@ -316,13 +303,7 @@ def login(self): self.results['actions'].append("Logged into %s" % (self.registry_url)) self.log("Log into %s with username %s" % (self.registry_url, self.username)) try: - response = self.client.login( - self.username, - password=self.password, - registry=self.registry_url, - reauth=self.reauthorize, - dockercfg_path=self.config_path - ) + response = self._login(self.reauthorize) except Exception as exc: self.fail("Logging into %s for user %s failed - %s" % (self.registry_url, self.username, to_native(exc))) @@ -333,13 +314,7 @@ def login(self): # reauthorize, still do it. if not self.reauthorize and response['password'] != self.password: try: - response = self.client.login( - self.username, - password=self.password, - registry=self.registry_url, - reauth=True, - dockercfg_path=self.config_path - ) + response = self._login(True) except Exception as exc: self.fail("Logging into %s for user %s failed - %s" % (self.registry_url, self.username, to_native(exc))) response.pop('password', None) @@ -358,7 +333,7 @@ def logout(self): store = self.get_credential_store_instance(self.registry_url, self.config_path) try: - current = store.get(self.registry_url) + store.get(self.registry_url) except CredentialsNotFound: # get raises an exception on not found. self.log("Credentials for %s not present, doing nothing." % (self.registry_url)) @@ -405,20 +380,11 @@ def get_credential_store_instance(self, registry, dockercfg_path): :rtype: Union[docker.credentials.Store, NoneType] ''' - # Older versions of docker-py don't have this feature. - try: - credstore_env = self.client.credstore_env - except AttributeError: - credstore_env = None + credstore_env = self.client.credstore_env config = auth.load_config(config_path=dockercfg_path) - if hasattr(auth, 'get_credential_store'): - store_name = auth.get_credential_store(config, registry) - elif 'credsStore' in config: - store_name = config['credsStore'] - else: - store_name = None + store_name = auth.get_credential_store(config, registry) # Make sure that there is a credential helper before trying to instantiate a # Store object. diff --git a/tests/integration/targets/docker_login/tasks/test.yml b/tests/integration/targets/docker_login/tasks/test.yml index 1135faec9..b60b70915 100644 --- a/tests/integration/targets/docker_login/tasks/test.yml +++ b/tests/integration/targets/docker_login/tasks/test.yml @@ -3,7 +3,7 @@ - include_tasks: run-test.yml with_fileglob: - "tests/*.yml" - when: docker_py_version is version('1.8.0', '>=') and docker_api_version is version('1.25', '>=') + when: docker_api_version is version('1.25', '>=') - fail: msg="Too old docker / docker-py version to run docker_image tests!" - when: not(docker_py_version is version('1.8.0', '>=') and docker_api_version is version('1.25', '>=')) and (ansible_distribution != 'CentOS' or ansible_distribution_major_version|int > 6) + when: not(docker_api_version is version('1.25', '>=')) and (ansible_distribution != 'CentOS' or ansible_distribution_major_version|int > 6) From 9afb672f4768c8c20875dbc86d9246c71ac94636 Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Sat, 25 Jun 2022 22:13:26 +0200 Subject: [PATCH 12/14] Adjust docker_api connection and docker_containers inventory plugins. --- plugins/connection/docker_api.py | 103 ++++++++++++++----------- plugins/inventory/docker_containers.py | 34 ++++---- 2 files changed, 74 insertions(+), 63 deletions(-) diff --git a/plugins/connection/docker_api.py b/plugins/connection/docker_api.py index e89cfc187..2cd4020a4 100644 --- a/plugins/connection/docker_api.py +++ b/plugins/connection/docker_api.py @@ -12,7 +12,7 @@ version_added: 1.1.0 description: - Run commands or put/fetch files to an existing docker container. - - Uses Docker SDK for Python to interact directly with the Docker daemon instead of + - Uses the Requests library to interact directly with the Docker daemon instead of using the Docker CLI. Use the R(community.docker.docker,ansible_collections.community.docker.docker_connection) connection plugin if you want to use the Docker CLI. @@ -64,9 +64,8 @@ type: integer extends_documentation_fragment: - - community.docker.docker + - community.docker.docker.api_documentation - community.docker.docker.var_names - - community.docker.docker.docker_py_1_documentation ''' import io @@ -80,23 +79,19 @@ from ansible.plugins.connection import ConnectionBase from ansible.utils.display import Display -from ansible_collections.community.docker.plugins.module_utils.common import ( +from ansible_collections.community.docker.plugins.module_utils.common_api import ( RequestException, ) from ansible_collections.community.docker.plugins.plugin_utils.socket_handler import ( DockerSocketHandler, ) -from ansible_collections.community.docker.plugins.plugin_utils.common import ( +from ansible_collections.community.docker.plugins.plugin_utils.common_api import ( AnsibleDockerClient, ) -try: - from docker.errors import DockerException, APIError, NotFound -except Exception: - # missing Docker SDK for Python handled in ansible_collections.community.docker.plugins.module_utils.common - pass +from ansible_collections.community.docker.plugins.module_utils._api.constants import DEFAULT_DATA_CHUNK_SIZE +from ansible_collections.community.docker.plugins.module_utils._api.errors import APIError, DockerException, NotFound -MIN_DOCKER_PY = '1.7.0' MIN_DOCKER_API = None @@ -154,7 +149,7 @@ def _connect(self, port=None): self.actual_user or u'?'), host=self.get_option('remote_addr') ) if self.client is None: - self.client = AnsibleDockerClient(self, min_docker_version=MIN_DOCKER_PY, min_docker_api_version=MIN_DOCKER_API) + self.client = AnsibleDockerClient(self, min_docker_api_version=MIN_DOCKER_API) self._connected = True if self.actual_user is None and display.verbosity > 2: @@ -162,7 +157,7 @@ def _connect(self, port=None): # Only do this if display verbosity is high enough that we'll need the value # This saves overhead from calling into docker when we don't need to display.vvv(u"Trying to determine actual user") - result = self._call_client(lambda: self.client.inspect_container(self.get_option('remote_addr'))) + result = self._call_client(lambda: self.client.get_json('/containers/{0}/json', self.get_option('remote_addr'))) if result.get('Config'): self.actual_user = result['Config'].get('User') if self.actual_user is not None: @@ -188,23 +183,29 @@ def exec_command(self, cmd, in_data=None, sudoable=False): need_stdin = True if (in_data is not None) or do_become else False - exec_data = self._call_client(lambda: self.client.exec_create( - self.get_option('remote_addr'), - command, - stdout=True, - stderr=True, - stdin=need_stdin, - user=self.get_option('remote_user') or '', - # workdir=None, - only works for Docker SDK for Python 3.0.0 and later - )) + data = { + 'Container': self.get_option('remote_addr'), + 'User': self.get_option('remote_user') or '', + 'Privileged': False, + 'Tty': False, + 'AttachStdin': need_stdin, + 'AttachStdout': True, + 'AttachStderr': True, + 'Cmd': command, + } + + if 'detachKeys' in self.client._general_configs: + data['detachKeys'] = self.client._general_configs['detachKeys'] + + exec_data = self._call_client(lambda: self.client.post_json_to_json('/containers/{0}/exec', self.get_option('remote_addr'), data=data)) exec_id = exec_data['Id'] + data = { + 'Tty': False, + 'Detach': False + } if need_stdin: - exec_socket = self._call_client(lambda: self.client.exec_start( - exec_id, - detach=False, - socket=True, - )) + exec_socket = self._call_client(lambda: self.client.post_json_to_stream_socket('/exec/{0}/start', exec_id, data=data)) try: with DockerSocketHandler(display, exec_socket, container=self.get_option('remote_addr')) as exec_socket_handler: if do_become: @@ -234,15 +235,10 @@ def append_become_output(stream_id, data): finally: exec_socket.close() else: - stdout, stderr = self._call_client(lambda: self.client.exec_start( - exec_id, - detach=False, - stream=False, - socket=False, - demux=True, - )) + stdout, stderr = self._call_client(lambda: self.client.post_json_to_stream( + '/exec/{0}/start', exec_id, stream=False, demux=True, tty=False, data=data)) - result = self._call_client(lambda: self.client.exec_inspect(exec_id)) + result = self._call_client(lambda: self.client.get_json('/exec/{0}/json', exec_id)) return result.get('ExitCode') or 0, stdout or b'', stderr or b'' @@ -264,6 +260,15 @@ def _prefix_login_path(self, remote_path): remote_path = os.path.join(os.path.sep, remote_path) return os.path.normpath(remote_path) + def _put_archive(self, container, path, data): + # data can also be file object for streaming. This is because _put uses requests's put(). + # See https://2.python-requests.org/en/master/user/advanced/#streaming-uploads + # WARNING: might not work with all transports! + url = self.client._url('/containers/{0}/archive', container) + res = self.client._put(url, params={'path': path}, data=data) + self.client._raise_for_status(res) + return res.status_code == 200 + def put_file(self, in_path, out_path): """ Transfer a file from local to docker container """ super(Connection, self).put_file(in_path, out_path) @@ -313,14 +318,14 @@ def put_file(self, in_path, out_path): tar.addfile(tarinfo, fileobj=f) data = bio.getvalue() - ok = self._call_client(lambda: self.client.put_archive( - self.get_option('remote_addr'), - out_dir, - data, # can also be file object for streaming; this is only clear from the - # implementation of put_archive(), which uses requests's put(). - # See https://2.python-requests.org/en/master/user/advanced/#streaming-uploads - # WARNING: might not work with all transports! - ), not_found_can_be_resource=True) + ok = self._call_client( + lambda: self._put_archive( + self.get_option('remote_addr'), + out_dir, + data, + ), + not_found_can_be_resource=True, + ) if not ok: raise AnsibleConnectionFailure( 'Unknown error while creating file "{0}" in container "{1}".' @@ -343,10 +348,14 @@ def fetch_file(self, in_path, out_path): considered_in_paths.add(in_path) display.vvvv('FETCH: Fetching "%s"' % in_path, host=self.get_option('remote_addr')) - stream, stats = self._call_client(lambda: self.client.get_archive( - self.get_option('remote_addr'), - in_path, - ), not_found_can_be_resource=True) + stream = self._call_client( + lambda: self.client.get_raw_stream( + '/containers/{0}/archive', self.get_option('remote_addr'), + params={'path': in_path}, + headers={'Accept-Encoding': 'identity'}, + ), + not_found_can_be_resource=True, + ) # TODO: stream tar file instead of downloading it into a BytesIO diff --git a/plugins/inventory/docker_containers.py b/plugins/inventory/docker_containers.py index ea1cca57f..a0aecb7c5 100644 --- a/plugins/inventory/docker_containers.py +++ b/plugins/inventory/docker_containers.py @@ -17,12 +17,9 @@ version_added: 1.1.0 author: - Felix Fontein (@felixfontein) -requirements: - - L(Docker SDK for Python,https://docker-py.readthedocs.io/en/stable/) >= 1.10.0 extends_documentation_fragment: - ansible.builtin.constructed - - community.docker.docker - - community.docker.docker.docker_py_1_documentation + - community.docker.docker.api_documentation description: - Reads inventories from the Docker API. - Uses a YAML configuration file that ends with C(docker.[yml|yaml]). @@ -154,23 +151,18 @@ from ansible.module_utils.common.text.converters import to_native from ansible.plugins.inventory import BaseInventoryPlugin, Constructable -from ansible_collections.community.docker.plugins.module_utils.common import ( +from ansible_collections.community.docker.plugins.module_utils.common_api import ( RequestException, ) from ansible_collections.community.docker.plugins.module_utils.util import ( DOCKER_COMMON_ARGS_VARS, ) -from ansible_collections.community.docker.plugins.plugin_utils.common import ( +from ansible_collections.community.docker.plugins.plugin_utils.common_api import ( AnsibleDockerClient, ) -try: - from docker.errors import DockerException, APIError -except Exception: - # missing Docker SDK for Python handled in ansible_collections.community.docker.plugins.module_utils.common - pass +from ansible_collections.community.docker.plugins.module_utils._api.errors import APIError, DockerException -MIN_DOCKER_PY = '1.7.0' MIN_DOCKER_API = None @@ -193,7 +185,15 @@ def _populate(self, client): add_legacy_groups = self.get_option('add_legacy_groups') try: - containers = client.containers(all=True) + params = { + 'limit': -1, + 'all': 1, + 'size': 0, + 'trunc_cmd': 0, + 'since': None, + 'before': None, + } + containers = client.get_json('/containers/json', params=params) except APIError as exc: raise AnsibleError("Error listing containers: %s" % to_native(exc)) @@ -227,7 +227,7 @@ def _populate(self, client): full_facts = dict() try: - inspect = client.inspect_container(id) + inspect = client.get_json('/containers/{0}/json', id) except APIError as exc: raise AnsibleError("Error inspecting container %s - %s" % (name, str(exc))) @@ -261,7 +261,9 @@ def _populate(self, client): # Figure out ssh IP and Port try: # Lookup the public facing port Nat'ed to ssh port. - port = client.port(container, ssh_port)[0] + network_settings = inspect.get('NetworkSettings') or {} + port_settings = network_settings.get('Ports') or {} + port = port_settings.get('%d/tcp' % (ssh_port, ))[0] except (IndexError, AttributeError, TypeError): port = dict() @@ -330,7 +332,7 @@ def verify_file(self, path): path.endswith(('docker.yaml', 'docker.yml'))) def _create_client(self): - return AnsibleDockerClient(self, min_docker_version=MIN_DOCKER_PY, min_docker_api_version=MIN_DOCKER_API) + return AnsibleDockerClient(self, min_docker_api_version=MIN_DOCKER_API) def parse(self, inventory, loader, path, cache=True): super(InventoryModule, self).parse(inventory, loader, path, cache) From 96f5b773ddcc6c3ec2d8b076d0480ae1d4e5e7f5 Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Sat, 25 Jun 2022 22:31:13 +0200 Subject: [PATCH 13/14] Adjust unit tests. --- .../inventory/test_docker_containers.py | 25 +++++++------------ 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/tests/unit/plugins/inventory/test_docker_containers.py b/tests/unit/plugins/inventory/test_docker_containers.py index b729d9bdd..46742daca 100644 --- a/tests/unit/plugins/inventory/test_docker_containers.py +++ b/tests/unit/plugins/inventory/test_docker_containers.py @@ -93,29 +93,22 @@ def get_option(option): class FakeClient(object): def __init__(self, *hosts): - self.hosts = dict() - self.list_reply = [] + self.get_results = {} + list_reply = [] for host in hosts: - self.list_reply.append({ + list_reply.append({ 'Id': host['Id'], 'Names': [host['Name']] if host['Name'] else [], 'Image': host['Config']['Image'], 'ImageId': host['Image'], }) - self.hosts[host['Name']] = host - self.hosts[host['Id']] = host + self.get_results['/containers/{0}/json'.format(host['Name'])] = host + self.get_results['/containers/{0}/json'.format(host['Id'])] = host + self.get_results['/containers/json'] = list_reply - def containers(self, all=False): - return list(self.list_reply) - - def inspect_container(self, id): - return self.hosts[id] - - def port(self, container, port): - host = self.hosts[container['Id']] - network_settings = host.get('NetworkSettings') or dict() - ports = network_settings.get('Ports') or dict() - return ports.get('{0}/tcp'.format(port)) or [] + def get_json(self, url, *param, **kwargs): + url = url.format(*param) + return self.get_results[url] def test_populate(inventory, mocker): From 18064f7d94d2db8b12770f1d7fc1f6ffcb930db1 Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Sat, 2 Jul 2022 17:49:56 +0200 Subject: [PATCH 14/14] Remove rebase artefact. --- plugins/module_utils/common_api.py | 193 ----------------------------- 1 file changed, 193 deletions(-) diff --git a/plugins/module_utils/common_api.py b/plugins/module_utils/common_api.py index a1f3f189a..da844d441 100644 --- a/plugins/module_utils/common_api.py +++ b/plugins/module_utils/common_api.py @@ -580,196 +580,3 @@ def report_warnings(self, result, warnings_key=None): self.module.warn('Docker warning: {0}'.format(warning)) elif isinstance(result, string_types) and result: self.module.warn('Docker warning: {0}'.format(result)) - - def get_container_by_id(self, container_id): - try: - self.log("Inspecting container Id %s" % container_id) - result = self.get_json('/containers/{0}/json', container_id) - self.log("Completed container inspection") - return result - except NotFound as dummy: - return None - except Exception as exc: - self.fail("Error inspecting container: %s" % exc) - - def get_container(self, name=None): - ''' - Lookup a container and return the inspection results. - ''' - if name is None: - return None - - search_name = name - if not name.startswith('/'): - search_name = '/' + name - - result = None - try: - params = { - 'limit': -1, - 'all': 1, - 'size': 0, - 'trunc_cmd': 0, - } - containers = self.get_json("/containers/json", params=params) - for container in containers: - self.log("testing container: %s" % (container['Names'])) - if isinstance(container['Names'], list) and search_name in container['Names']: - result = container - break - if container['Id'].startswith(name): - result = container - break - if container['Id'] == name: - result = container - break - except SSLError as exc: - self._handle_ssl_error(exc) - except Exception as exc: - self.fail("Error retrieving container list: %s" % exc) - - if result is None: - return None - - return self.get_container_by_id(result['Id']) - - def get_network(self, name=None, network_id=None): - ''' - Lookup a network and return the inspection results. - ''' - if name is None and network_id is None: - return None - - result = None - - if network_id is None: - try: - networks = self.get_json("/networks") - for network in networks: - self.log("testing network: %s" % (network['Name'])) - if name == network['Name']: - result = network - break - if network['Id'].startswith(name): - result = network - break - except SSLError as exc: - self._handle_ssl_error(exc) - except Exception as exc: - self.fail("Error retrieving network list: %s" % exc) - - if result is not None: - network_id = result['Id'] - - if network_id is not None: - try: - self.log("Inspecting network Id %s" % network_id) - result = self.get_json('/networks/{0}', network_id) - self.log("Completed network inspection") - except NotFound as dummy: - return None - except Exception as exc: - self.fail("Error inspecting network: %s" % exc) - - return result - - def _image_lookup(self, name, tag): - ''' - Including a tag in the name parameter sent to the Docker SDK for Python images method - does not work consistently. Instead, get the result set for name and manually check - if the tag exists. - ''' - try: - params = { - 'only_ids': 0, - 'all': 0, - } - if LooseVersion(self.api_version) < LooseVersion('1.25'): - # only use "filter" on API 1.24 and under, as it is deprecated - params['filter'] = name - else: - params['filters'] = convert_filters({'reference': name}) - images = self.get_json("/images/json", params=params) - except Exception as exc: - self.fail("Error searching for image %s - %s" % (name, str(exc))) - if tag: - lookup = "%s:%s" % (name, tag) - lookup_digest = "%s@%s" % (name, tag) - response = images - images = [] - for image in response: - tags = image.get('RepoTags') - digests = image.get('RepoDigests') - if (tags and lookup in tags) or (digests and lookup_digest in digests): - images = [image] - break - return images - - def find_image(self, name, tag): - ''' - Lookup an image (by name and tag) and return the inspection results. - ''' - if not name: - return None - - self.log("Find image %s:%s" % (name, tag)) - images = self._image_lookup(name, tag) - if not images: - # In API <= 1.20 seeing 'docker.io/' as the name of images pulled from docker hub - registry, repo_name = auth.resolve_repository_name(name) - if registry == 'docker.io': - # If docker.io is explicitly there in name, the image - # isn't found in some cases (#41509) - self.log("Check for docker.io image: %s" % repo_name) - images = self._image_lookup(repo_name, tag) - if not images and repo_name.startswith('library/'): - # Sometimes library/xxx images are not found - lookup = repo_name[len('library/'):] - self.log("Check for docker.io image: %s" % lookup) - images = self._image_lookup(lookup, tag) - if not images: - # Last case for some Docker versions: if docker.io wasn't there, - # it can be that the image wasn't found either - # (https://github.com/ansible/ansible/pull/15586) - lookup = "%s/%s" % (registry, repo_name) - self.log("Check for docker.io image: %s" % lookup) - images = self._image_lookup(lookup, tag) - if not images and '/' not in repo_name: - # This seems to be happening with podman-docker - # (https://github.com/ansible-collections/community.docker/issues/291) - lookup = "%s/library/%s" % (registry, repo_name) - self.log("Check for docker.io image: %s" % lookup) - images = self._image_lookup(lookup, tag) - - if len(images) > 1: - self.fail("Registry returned more than one result for %s:%s" % (name, tag)) - - if len(images) == 1: - try: - return self.get_json('/images/{0}/json', images[0]['Id']) - except NotFound: - self.log("Image %s:%s not found." % (name, tag)) - return None - except Exception as exc: - self.fail("Error inspecting image %s:%s - %s" % (name, tag, str(exc))) - - self.log("Image %s:%s not found." % (name, tag)) - return None - - def find_image_by_id(self, image_id, accept_missing_image=False): - ''' - Lookup an image (by ID) and return the inspection results. - ''' - if not image_id: - return None - - self.log("Find image %s (by ID)" % image_id) - try: - return self.get_json('/images/{0}/json', image_id) - except NotFound as exc: - if not accept_missing_image: - self.fail("Error inspecting image ID %s - %s" % (image_id, str(exc))) - self.log("Image %s not found." % image_id) - return None - except Exception as exc: - self.fail("Error inspecting image ID %s - %s" % (image_id, str(exc)))