[WILL BE SPLIT UP] Start adjusting some modules

changelogs/fragments/387-docker-api.yml

@@ -0,0 +1,8 @@
+major_changes:
+  - "The collection now contains vendored code from the Docker SDK for Python to talk to the Docker daemon.
+     Modules and plugins using this code no longer need the Docker SDK for Python installed on the machine
+     the module resp. plugin is running on
+     (https://github.com/ansible-collections/community.docker/pull/387)."
+  - "docker_host_info - no longer uses the Docker SDK for Python. It requires ``requests`` to be installed,
+     and depending on the features used has some more requirements. If the Docker SDK for Python is installed,
+     these requirements are likely met (https://github.com/ansible-collections/community.docker/pull/387)."

plugins/connection/docker_api.py

@@ -12,7 +12,7 @@
 version_added: 1.1.0
 description:
     - Run commands or put/fetch files to an existing docker container.
-    - Uses Docker SDK for Python to interact directly with the Docker daemon instead of
+    - Uses the Requests library to interact directly with the Docker daemon instead of
       using the Docker CLI. Use the
       R(community.docker.docker,ansible_collections.community.docker.docker_connection)
       connection plugin if you want to use the Docker CLI.
@@ -64,9 +64,8 @@
         type: integer
 
 extends_documentation_fragment:
-    - community.docker.docker
+    - community.docker.docker.api_documentation
     - community.docker.docker.var_names
-    - community.docker.docker.docker_py_1_documentation
 '''
 
 import io
@@ -80,23 +79,19 @@
 from ansible.plugins.connection import ConnectionBase
 from ansible.utils.display import Display
 
-from ansible_collections.community.docker.plugins.module_utils.common import (
+from ansible_collections.community.docker.plugins.module_utils.common_api import (
     RequestException,
 )
 from ansible_collections.community.docker.plugins.plugin_utils.socket_handler import (
     DockerSocketHandler,
 )
-from ansible_collections.community.docker.plugins.plugin_utils.common import (
+from ansible_collections.community.docker.plugins.plugin_utils.common_api import (
     AnsibleDockerClient,
 )
 
-try:
-    from docker.errors import DockerException, APIError, NotFound
-except Exception:
-    # missing Docker SDK for Python handled in ansible_collections.community.docker.plugins.module_utils.common
-    pass
+from ansible_collections.community.docker.plugins.module_utils._api.constants import DEFAULT_DATA_CHUNK_SIZE
+from ansible_collections.community.docker.plugins.module_utils._api.errors import APIError, DockerException, NotFound
 
-MIN_DOCKER_PY = '1.7.0'
 MIN_DOCKER_API = None
 
 
@@ -154,15 +149,15 @@ def _connect(self, port=None):
                 self.actual_user or u'?'), host=self.get_option('remote_addr')
             )
             if self.client is None:
-                self.client = AnsibleDockerClient(self, min_docker_version=MIN_DOCKER_PY, min_docker_api_version=MIN_DOCKER_API)
+                self.client = AnsibleDockerClient(self, min_docker_api_version=MIN_DOCKER_API)
             self._connected = True
 
             if self.actual_user is None and display.verbosity > 2:
                 # Since we're not setting the actual_user, look it up so we have it for logging later
                 # Only do this if display verbosity is high enough that we'll need the value
                 # This saves overhead from calling into docker when we don't need to
                 display.vvv(u"Trying to determine actual user")
-                result = self._call_client(lambda: self.client.inspect_container(self.get_option('remote_addr')))
+                result = self._call_client(lambda: self.client.get_json('/containers/{0}/json', self.get_option('remote_addr')))
                 if result.get('Config'):
                     self.actual_user = result['Config'].get('User')
                     if self.actual_user is not None:
@@ -188,23 +183,29 @@ def exec_command(self, cmd, in_data=None, sudoable=False):
 
         need_stdin = True if (in_data is not None) or do_become else False
 
-        exec_data = self._call_client(lambda: self.client.exec_create(
-            self.get_option('remote_addr'),
-            command,
-            stdout=True,
-            stderr=True,
-            stdin=need_stdin,
-            user=self.get_option('remote_user') or '',
-            # workdir=None,  - only works for Docker SDK for Python 3.0.0 and later
-        ))
+        data = {
+            'Container': self.get_option('remote_addr'),
+            'User': self.get_option('remote_user') or '',
+            'Privileged': False,
+            'Tty': False,
+            'AttachStdin': need_stdin,
+            'AttachStdout': True,
+            'AttachStderr': True,
+            'Cmd': command,
+        }
+
+        if 'detachKeys' in self.client._general_configs:
+            data['detachKeys'] = self.client._general_configs['detachKeys']
+
+        exec_data = self._call_client(lambda: self.client.post_json_to_json('/containers/{0}/exec', self.get_option('remote_addr'), data=data))
         exec_id = exec_data['Id']
 
+        data = {
+            'Tty': False,
+            'Detach': False
+        }
         if need_stdin:
-            exec_socket = self._call_client(lambda: self.client.exec_start(
-                exec_id,
-                detach=False,
-                socket=True,
-            ))
+            exec_socket = self._call_client(lambda: self.client.post_json_to_stream_socket('/exec/{0}/start', exec_id, data=data))
             try:
                 with DockerSocketHandler(display, exec_socket, container=self.get_option('remote_addr')) as exec_socket_handler:
                     if do_become:
@@ -234,15 +235,10 @@ def append_become_output(stream_id, data):
             finally:
                 exec_socket.close()
         else:
-            stdout, stderr = self._call_client(lambda: self.client.exec_start(
-                exec_id,
-                detach=False,
-                stream=False,
-                socket=False,
-                demux=True,
-            ))
+            stdout, stderr = self._call_client(lambda: self.client.post_json_to_stream(
+                '/exec/{0}/start', exec_id, stream=False, demux=True, tty=False, data=data))
 
-        result = self._call_client(lambda: self.client.exec_inspect(exec_id))
+        result = self._call_client(lambda: self.client.get_json('/exec/{0}/json', exec_id))
 
         return result.get('ExitCode') or 0, stdout or b'', stderr or b''
 
@@ -264,6 +260,15 @@ def _prefix_login_path(self, remote_path):
                 remote_path = os.path.join(os.path.sep, remote_path)
             return os.path.normpath(remote_path)
 
+    def _put_archive(self, container, path, data):
+        # data can also be file object for streaming. This is because _put uses requests's put().
+        # See https://2.python-requests.org/en/master/user/advanced/#streaming-uploads
+        # WARNING: might not work with all transports!
+        url = self.client._url('/containers/{0}/archive', container)
+        res = self.client._put(url, params={'path': path}, data=data)
+        self.client._raise_for_status(res)
+        return res.status_code == 200
+
     def put_file(self, in_path, out_path):
         """ Transfer a file from local to docker container """
         super(Connection, self).put_file(in_path, out_path)
@@ -313,14 +318,14 @@ def put_file(self, in_path, out_path):
                 tar.addfile(tarinfo, fileobj=f)
         data = bio.getvalue()
 
-        ok = self._call_client(lambda: self.client.put_archive(
-            self.get_option('remote_addr'),
-            out_dir,
-            data,  # can also be file object for streaming; this is only clear from the
-                   # implementation of put_archive(), which uses requests's put().
-                   # See https://2.python-requests.org/en/master/user/advanced/#streaming-uploads
-                   # WARNING: might not work with all transports!
-        ), not_found_can_be_resource=True)
+        ok = self._call_client(
+            lambda: self._put_archive(
+                self.get_option('remote_addr'),
+                out_dir,
+                data,
+            ),
+            not_found_can_be_resource=True,
+        )
         if not ok:
             raise AnsibleConnectionFailure(
                 'Unknown error while creating file "{0}" in container "{1}".'
@@ -343,10 +348,14 @@ def fetch_file(self, in_path, out_path):
             considered_in_paths.add(in_path)
 
             display.vvvv('FETCH: Fetching "%s"' % in_path, host=self.get_option('remote_addr'))
-            stream, stats = self._call_client(lambda: self.client.get_archive(
-                self.get_option('remote_addr'),
-                in_path,
-            ), not_found_can_be_resource=True)
+            stream = self._call_client(
+                lambda: self.client.get_raw_stream(
+                    '/containers/{0}/archive', self.get_option('remote_addr'),
+                    params={'path': in_path},
+                    headers={'Accept-Encoding': 'identity'},
+                ),
+                not_found_can_be_resource=True,
+            )
 
             # TODO: stream tar file instead of downloading it into a BytesIO
 

plugins/inventory/docker_containers.py

@@ -17,12 +17,9 @@
 version_added: 1.1.0
 author:
     - Felix Fontein (@felixfontein)
-requirements:
-    - L(Docker SDK for Python,https://docker-py.readthedocs.io/en/stable/) >= 1.10.0
 extends_documentation_fragment:
     - ansible.builtin.constructed
-    - community.docker.docker
-    - community.docker.docker.docker_py_1_documentation
+    - community.docker.docker.api_documentation
 description:
     - Reads inventories from the Docker API.
     - Uses a YAML configuration file that ends with C(docker.[yml|yaml]).
@@ -154,23 +151,18 @@
 from ansible.module_utils.common.text.converters import to_native
 from ansible.plugins.inventory import BaseInventoryPlugin, Constructable
 
-from ansible_collections.community.docker.plugins.module_utils.common import (
+from ansible_collections.community.docker.plugins.module_utils.common_api import (
     RequestException,
 )
 from ansible_collections.community.docker.plugins.module_utils.util import (
     DOCKER_COMMON_ARGS_VARS,
 )
-from ansible_collections.community.docker.plugins.plugin_utils.common import (
+from ansible_collections.community.docker.plugins.plugin_utils.common_api import (
     AnsibleDockerClient,
 )
 
-try:
-    from docker.errors import DockerException, APIError
-except Exception:
-    # missing Docker SDK for Python handled in ansible_collections.community.docker.plugins.module_utils.common
-    pass
+from ansible_collections.community.docker.plugins.module_utils._api.errors import APIError, DockerException
 
-MIN_DOCKER_PY = '1.7.0'
 MIN_DOCKER_API = None
 
 
@@ -193,7 +185,15 @@ def _populate(self, client):
         add_legacy_groups = self.get_option('add_legacy_groups')
 
         try:
-            containers = client.containers(all=True)
+            params = {
+                'limit': -1,
+                'all': 1,
+                'size': 0,
+                'trunc_cmd': 0,
+                'since': None,
+                'before': None,
+            }
+            containers = client.get_json('/containers/json', params=params)
         except APIError as exc:
             raise AnsibleError("Error listing containers: %s" % to_native(exc))
 
@@ -227,7 +227,7 @@ def _populate(self, client):
             full_facts = dict()
 
             try:
-                inspect = client.inspect_container(id)
+                inspect = client.get_json('/containers/{0}/json', id)
             except APIError as exc:
                 raise AnsibleError("Error inspecting container %s - %s" % (name, str(exc)))
 
@@ -261,7 +261,9 @@ def _populate(self, client):
                 # Figure out ssh IP and Port
                 try:
                     # Lookup the public facing port Nat'ed to ssh port.
-                    port = client.port(container, ssh_port)[0]
+                    network_settings = inspect.get('NetworkSettings') or {}
+                    port_settings = network_settings.get('Ports') or {}
+                    port = port_settings.get('%d/tcp' % (ssh_port, ))[0]
                 except (IndexError, AttributeError, TypeError):
                     port = dict()
 
@@ -330,7 +332,7 @@ def verify_file(self, path):
             path.endswith(('docker.yaml', 'docker.yml')))
 
     def _create_client(self):
-        return AnsibleDockerClient(self, min_docker_version=MIN_DOCKER_PY, min_docker_api_version=MIN_DOCKER_API)
+        return AnsibleDockerClient(self, min_docker_api_version=MIN_DOCKER_API)
 
     def parse(self, inventory, loader, path, cache=True):
         super(InventoryModule, self).parse(inventory, loader, path, cache)