Add docker_container_copy_into module

README.md

@@ -57,6 +57,7 @@ If you use the Ansible package and do not update collections independently, use
 * Modules:
   * Docker:
     - community.docker.docker_container: manage Docker containers
+    - community.docker.docker_container_copy_into: copy a file into a Docker container
     - community.docker.docker_container_exec: run commands in Docker containers
     - community.docker.docker_container_info: retrieve information on Docker containers
     - community.docker.docker_host_info: retrieve information on the Docker daemon

changelogs/fragments/545-docker_api.yml

@@ -0,0 +1,2 @@
+minor_changes:
+  - "docker_api connection plugin - when copying files to/from a container, stream the file contents instead of first reading them to memory (https://github.com/ansible-collections/community.docker/pull/545)."

meta/runtime.yml

@@ -9,6 +9,7 @@ action_groups:
     - docker_compose
     - docker_config
     - docker_container
+    - docker_container_copy_into
     - docker_container_exec
     - docker_container_info
     - docker_host_info

plugins/action/docker_container_copy_into.py

@@ -0,0 +1,41 @@
+# Copyright (c) 2022, Felix Fontein <[email protected]>
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from __future__ import absolute_import, division, print_function
+__metaclass__ = type
+
+import base64
+
+from ansible import constants as C
+from ansible.errors import AnsibleError
+from ansible.plugins.action import ActionBase
+from ansible.utils.vars import merge_hash
+
+from ansible_collections.community.docker.plugins.module_utils._scramble import unscramble
+
+
+class ActionModule(ActionBase):
+    # Set to True when transfering files to the remote
+    TRANSFERS_FILES = False
+
+    def run(self, tmp=None, task_vars=None):
+        self._supports_check_mode = True
+        self._supports_async = True
+
+        result = super(ActionModule, self).run(tmp, task_vars)
+        del tmp  # tmp no longer has any effect
+
+        self._task.args['_max_file_size_for_diff'] = C.MAX_FILE_SIZE_FOR_DIFF
+
+        result = merge_hash(result, self._execute_module(task_vars=task_vars, wrap_async=self._task.async_val))
+
+        if u'diff' in result and result[u'diff'].get(u'scrambled_diff'):
+            # Scrambling is not done for security, but to avoid no_log screwing up the diff
+            diff = result[u'diff']
+            key = base64.b64decode(diff.pop(u'scrambled_diff'))
+            for k in (u'before', u'after'):
+                if k in diff:
+                    diff[k] = unscramble(diff[k], key)
+
+        return result

plugins/connection/docker_api.py

@@ -68,11 +68,8 @@
         type: integer
 '''
 
-import io
 import os
 import os.path
-import shutil
-import tarfile
 
 from ansible.errors import AnsibleFileNotFound, AnsibleConnectionFailure
 from ansible.module_utils.common.text.converters import to_bytes, to_native, to_text
@@ -82,14 +79,20 @@
 from ansible_collections.community.docker.plugins.module_utils.common_api import (
     RequestException,
 )
+from ansible_collections.community.docker.plugins.module_utils.copy import (
+    DockerFileCopyError,
+    DockerFileNotFound,
+    fetch_file,
+    put_file,
+)
+
 from ansible_collections.community.docker.plugins.plugin_utils.socket_handler import (
     DockerSocketHandler,
 )
 from ansible_collections.community.docker.plugins.plugin_utils.common_api import (
     AnsibleDockerClient,
 )
 
-from ansible_collections.community.docker.plugins.module_utils._api.constants import DEFAULT_DATA_CHUNK_SIZE
 from ansible_collections.community.docker.plugins.module_utils._api.errors import APIError, DockerException, NotFound
 
 MIN_DOCKER_API = None
@@ -260,24 +263,12 @@ def _prefix_login_path(self, remote_path):
                 remote_path = os.path.join(os.path.sep, remote_path)
             return os.path.normpath(remote_path)
 
-    def _put_archive(self, container, path, data):
-        # data can also be file object for streaming. This is because _put uses requests's put().
-        # See https://2.python-requests.org/en/master/user/advanced/#streaming-uploads
-        # WARNING: might not work with all transports!
-        url = self.client._url('/containers/{0}/archive', container)
-        res = self.client._put(url, params={'path': path}, data=data)
-        self.client._raise_for_status(res)
-        return res.status_code == 200
-
     def put_file(self, in_path, out_path):
         """ Transfer a file from local to docker container """
         super(Connection, self).put_file(in_path, out_path)
         display.vvv("PUT %s TO %s" % (in_path, out_path), host=self.get_option('remote_addr'))
 
         out_path = self._prefix_login_path(out_path)
-        if not os.path.exists(to_bytes(in_path, errors='surrogate_or_strict')):
-            raise AnsibleFileNotFound(
-                "file or module does not exist: %s" % to_native(in_path))
 
         if self.actual_user not in self.ids:
             dummy, ids, dummy = self.exec_command(b'id -u && id -g')
@@ -294,99 +285,49 @@ def put_file(self, in_path, out_path):
                     .format(e, self.get_option('remote_addr'), ids)
                 )
 
-        b_in_path = to_bytes(in_path, errors='surrogate_or_strict')
-
-        out_dir, out_file = os.path.split(out_path)
-
-        # TODO: stream tar file, instead of creating it in-memory into a BytesIO
-
-        bio = io.BytesIO()
-        with tarfile.open(fileobj=bio, mode='w|', dereference=True, encoding='utf-8') as tar:
-            # Note that without both name (bytes) and arcname (unicode), this either fails for
-            # Python 2.7, Python 3.5/3.6, or Python 3.7+. Only when passing both (in this
-            # form) it works with Python 2.7, 3.5, 3.6, and 3.7 up to 3.11
-            tarinfo = tar.gettarinfo(b_in_path, arcname=to_text(out_file))
-            user_id, group_id = self.ids[self.actual_user]
-            tarinfo.uid = user_id
-            tarinfo.uname = ''
-            if self.actual_user:
-                tarinfo.uname = self.actual_user
-            tarinfo.gid = group_id
-            tarinfo.gname = ''
-            tarinfo.mode &= 0o700
-            with open(b_in_path, 'rb') as f:
-                tar.addfile(tarinfo, fileobj=f)
-        data = bio.getvalue()
-
-        ok = self._call_client(
-            lambda: self._put_archive(
-                self.get_option('remote_addr'),
-                out_dir,
-                data,
-            ),
-            not_found_can_be_resource=True,
-        )
-        if not ok:
-            raise AnsibleConnectionFailure(
-                'Unknown error while creating file "{0}" in container "{1}".'
-                .format(out_path, self.get_option('remote_addr'))
+        user_id, group_id = self.ids[self.actual_user]
+        try:
+            self._call_client(
+                lambda: put_file(
+                    self.client,
+                    container=self.get_option('remote_addr'),
+                    in_path=in_path,
+                    out_path=out_path,
+                    user_id=user_id,
+                    group_id=group_id,
+                    user_name=self.actual_user,
+                    follow_links=True,
+                ),
+                not_found_can_be_resource=True,
             )
+        except DockerFileNotFound as exc:
+            raise AnsibleFileNotFound(to_native(exc))
+        except DockerFileCopyError as exc:
+            raise AnsibleConnectionFailure(to_native(exc))
 
     def fetch_file(self, in_path, out_path):
         """ Fetch a file from container to local. """
         super(Connection, self).fetch_file(in_path, out_path)
         display.vvv("FETCH %s TO %s" % (in_path, out_path), host=self.get_option('remote_addr'))
 
         in_path = self._prefix_login_path(in_path)
-        b_out_path = to_bytes(out_path, errors='surrogate_or_strict')
-
-        considered_in_paths = set()
 
-        while True:
-            if in_path in considered_in_paths:
-                raise AnsibleConnectionFailure('Found infinite symbolic link loop when trying to fetch "{0}"'.format(in_path))
-            considered_in_paths.add(in_path)
-
-            display.vvvv('FETCH: Fetching "%s"' % in_path, host=self.get_option('remote_addr'))
-            stream = self._call_client(
-                lambda: self.client.get_raw_stream(
-                    '/containers/{0}/archive', self.get_option('remote_addr'),
-                    params={'path': in_path},
-                    headers={'Accept-Encoding': 'identity'},
+        try:
+            self._call_client(
+                lambda: fetch_file(
+                    self.client,
+                    container=self.get_option('remote_addr'),
+                    in_path=in_path,
+                    out_path=out_path,
+                    follow_links=True,
+                    log=lambda msg: display.vvvv(msg, host=self.get_option('remote_addr')),
                 ),
                 not_found_can_be_resource=True,
             )
-
-            # TODO: stream tar file instead of downloading it into a BytesIO
-
-            bio = io.BytesIO()
-            for chunk in stream:
-                bio.write(chunk)
-            bio.seek(0)
-
-            with tarfile.open(fileobj=bio, mode='r|') as tar:
-                symlink_member = None
-                first = True
-                for member in tar:
-                    if not first:
-                        raise AnsibleConnectionFailure('Received tarfile contains more than one file!')
-                    first = False
-                    if member.issym():
-                        symlink_member = member
-                        continue
-                    if not member.isfile():
-                        raise AnsibleConnectionFailure('Remote file "%s" is not a regular file or a symbolic link' % in_path)
-                    in_f = tar.extractfile(member)  # in Python 2, this *cannot* be used in `with`...
-                    with open(b_out_path, 'wb') as out_f:
-                        shutil.copyfileobj(in_f, out_f, member.size)
-                if first:
-                    raise AnsibleConnectionFailure('Received tarfile is empty!')
-                # If the only member was a file, it's already extracted. If it is a symlink, process it now.
-                if symlink_member is not None:
-                    in_path = os.path.join(os.path.split(in_path)[0], symlink_member.linkname)
-                    display.vvvv('FETCH: Following symbolic link to "%s"' % in_path, host=self.get_option('remote_addr'))
-                    continue
-                return
+        except DockerFileNotFound as exc:
+            raise AnsibleFileNotFound(to_native(exc))
+        except DockerFileCopyError as exc:
+            raise AnsibleConnectionFailure(to_native(exc))
 
     def close(self):
         """ Terminate the connection. Nothing to do for Docker"""

plugins/module_utils/_api/api/client.py

@@ -227,6 +227,10 @@ def _post(self, url, **kwargs):
     def _get(self, url, **kwargs):
         return self.get(url, **self._set_request_timeout(kwargs))
 
+    @update_headers
+    def _head(self, url, **kwargs):
+        return self.head(url, **self._set_request_timeout(kwargs))
+
     @update_headers
     def _put(self, url, **kwargs):
         return self.put(url, **self._set_request_timeout(kwargs))

plugins/module_utils/_scramble.py

@@ -0,0 +1,56 @@
+# Copyright 2016 Red Hat | Ansible
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import base64
+import random
+
+from ansible.module_utils.common.text.converters import to_bytes, to_native, to_text
+from ansible.module_utils.six import PY2
+
+
+def generate_insecure_key():
+    '''Do NOT use this for cryptographic purposes!'''
+    while True:
+        # Generate a one-byte key. Right now the functions below do not use more
+        # than one byte, so this is sufficient.
+        if PY2:
+            key = chr(random.randint(0, 255))
+        else:
+            key = bytes([random.randint(0, 255)])
+        # Return anything that is not zero
+        if key != b'\x00':
+            return key
+
+
+def scramble(value, key):
+    '''Do NOT use this for cryptographic purposes!'''
+    if len(key) < 1:
+        raise ValueError('Key must be at least one byte')
+    value = to_bytes(value)
+    if PY2:
+        k = ord(key[0])
+        value = b''.join([chr(k ^ ord(b)) for b in value])
+    else:
+        k = key[0]
+        value = bytes([k ^ b for b in value])
+    return '=S=' + to_native(base64.b64encode(value))
+
+
+def unscramble(value, key):
+    '''Do NOT use this for cryptographic purposes!'''
+    if len(key) < 1:
+        raise ValueError('Key must be at least one byte')
+    if not value.startswith(u'=S='):
+        raise ValueError('Value does not start with indicator')
+    value = base64.b64decode(value[3:])
+    if PY2:
+        k = ord(key[0])
+        value = b''.join([chr(k ^ ord(b)) for b in value])
+    else:
+        k = key[0]
+        value = bytes([k ^ b for b in value])
+    return to_text(value)