From 315161c7e222293015bb27989dbf193aeea2cf5c Mon Sep 17 00:00:00 2001 From: "Owen W. Taylor" Date: Fri, 8 Jun 2018 12:00:09 -0400 Subject: [PATCH] Add bodhi-skopeo-lite - a skopeo-workalike with manifest list support In order to support copying multi-arch containers and Flatpaks, we need to be able to copy manifest lists and OCI image indexes from registry to registry. Work is underway to add such support to skopeo (https://github.com/containers/image/pull/400), but as a temporary workaround add 'bodhi-skopeo-lite', which implements the subset of 'skopeo copy' we need, but with manifest list/image index support. Use of this needs to be specifically configured. Signed-off-by: Owen W. Taylor --- bodhi/server/scripts/skopeo_lite.py | 777 ++++++++++++++++++ .../tests/server/scripts/test_skopeo_lite.py | 732 +++++++++++++++++ devel/ansible/roles/dev/tasks/main.yml | 2 + devel/ci/pip-packages | 2 + devel/ci/rpm-packages | 2 + docs/conf.py | 3 + docs/user/man_pages/bodhi-skopeo-lite.rst | 79 ++ docs/user/man_pages/index.rst | 1 + docs/user/release_notes.rst | 1 + production.ini | 7 + setup.py | 1 + 11 files changed, 1607 insertions(+) create mode 100644 bodhi/server/scripts/skopeo_lite.py create mode 100644 bodhi/tests/server/scripts/test_skopeo_lite.py create mode 100644 docs/user/man_pages/bodhi-skopeo-lite.rst diff --git a/bodhi/server/scripts/skopeo_lite.py b/bodhi/server/scripts/skopeo_lite.py new file mode 100644 index 0000000000..df0db01329 --- /dev/null +++ b/bodhi/server/scripts/skopeo_lite.py @@ -0,0 +1,777 @@ +# -*- coding: utf-8 -*- +# Copyright © 2018 Red Hat, Inc. +# +# This file is part of Bodhi. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +""" +Copy containers between registries. + +This is a very limited version of the skopeo tool, but with support +for manifests lists and OCI image indexes. +https://github.com/containers/image/pull/400 will make this +unnecessary. + +The only subcommand that is supported is 'copy', and the only supported image references +are Docker registry references of the form 'docker://docker-reference'. + +No global options are supported, and only selected options to 'copy' are supported (see +--help for details.) + +Some other things that aren't implemented (but could be added if necessary): + - Handling of www-authenticate responses, necessary to log in to docker.io + - Special handling of 'docker.io' as 'registry-1.docker.io' + - Reading ~/.docker/config.json or $XDG_RUNTIME_DIR/containers/auth.json + - Handling foreign layers +""" + +import json +import logging +import os +import shutil +import tempfile + +import click +import requests +from requests.exceptions import SSLError, ConnectionError +from six.moves.urllib.parse import urlparse, urlunparse + + +@click.group() +def main(): + """Simplified Skopeo work-alike with manifest list support.""" + pass # pragma: no cover + + +logger = logging.getLogger('skopeo-lite') +logging.basicConfig(level=logging.INFO) + + +MEDIA_TYPE_MANIFEST_V2 = 'application/vnd.docker.distribution.manifest.v2+json' +MEDIA_TYPE_LIST_V2 = 'application/vnd.docker.distribution.manifest.list.v2+json' +MEDIA_TYPE_OCI = 'application/vnd.oci.image.manifest.v1+json' +MEDIA_TYPE_OCI_INDEX = 'application/vnd.oci.image.index.v1+json' + + +class RegistrySpec(object): + """Information about a docker registry/repository/tag as specified on the command line.""" + + def __init__(self, registry, repo, tag, creds, tls_verify, cert_dir): + """ + Initialize the registry spec. + + Args: + registry (str): The hostname. + repo (str): The repository name within the registry. + tag (str): The tag within the repository. + creds (str): user:password (may be None). + tls_verify (bool): If True, HTTPS with a verified certificate is required. + cert_dir (str): A path to directory holding client certificates, or None. + """ + self.registry = registry + self.repo = repo + self.tag = tag + self.creds = creds + self.cert_dir = cert_dir + self.tls_verify = tls_verify + + def get_session(self): + """Create a RegistrySession object for the spec.""" + return RegistrySession(self.registry, insecure=not self.tls_verify, + creds=self.creds, cert_dir=self.cert_dir) + + def get_endpoint(self): + """Create a RegistryEndpoint object for the spec.""" + return RegistryEndpoint(self) + + +def parse_spec(spec, creds, tls_verify, cert_dir): + """ + Parse a string into a RegistrySpec, adding extra information. + + Args: + spec (str): docker:///[:tag] - latest will be used + if tag is omitted. + creds (str): user:password (may be None). + tls_verify (bool): If True https with a verified certificate is required + cert_dir (str): A path to directory holding client certificates, or None. + Returns: + RegistrySpec: The resulting registry spec. + Raises: + click.BadArgumentUsage: If the string cannot be parsed + """ + if spec.startswith('docker:'): + if not spec.startswith('docker://'): + raise click.BadArgumentUsage( + "Registry specification should be docker://REGISTRY/PATH[:TAG]") + + parts = spec[len('docker://'):].split('/', 1) + if len(parts) == 1: + raise click.BadArgumentUsage( + "Registry specification should be docker://REGISTRY/PATH[:TAG]") + + registry, path = parts + parts = path.split(':', 1) + if len(parts) == 1: + repo, tag = parts[0], 'latest' + else: + repo, tag = parts + + return RegistrySpec(registry, repo, tag, creds, tls_verify, cert_dir) + else: + raise click.BadArgumentUsage("Unknown source/destination: {}".format(spec)) + + +class RegistrySession(object): + """Wrapper around requests.Session adding docker-specific behavior.""" + + def __init__(self, registry, insecure=False, creds=None, cert_dir=None): + """ + Initialize the RegistrySession. + + Args: + registry (str): The hostname of the registry. + insecure (bool): If True don't verify TLS certificates and fallback to HTTP. + creds (str): user:password (may be None). + cert_dir (str): A path to directory holding client certificates, or None. + """ + self.registry = registry + self._resolved = None + self.insecure = insecure + + self.cert = self._find_cert(cert_dir) + + self.auth = None + if creds is not None: + username, password = creds.split(':', 1) + self.auth = requests.auth.HTTPBasicAuth(username, password) + + self._fallback = None + self._base = 'https://{}'.format(self.registry) + if insecure: + # In the insecure case, if the registry is just a hostname:port, we + # don't know whether to talk HTTPS or HTTP to it, so we try first + # with https then fallback + self._fallback = 'http://{}'.format(self.registry) + + self.session = requests.Session() + + def _find_cert_dir(self): + """ + Return a path to a directory containing TLS client certificates to use for authentication. + + Returns: + str or None: If a path is found, it is returned. Otherwise None is returned. + """ + hostport = self.registry + + for d in ('/etc/containers/certs.d', '/etc/docker/certs.d'): + certs_dir = os.path.join(d, hostport) + if os.path.isdir(certs_dir): + return certs_dir + + return None + + def _find_cert(self, cert_dir): + """ + Return a TLS client certificate to be used to authenticate to servers. + + Args: + cert_dir (str or None): A directory to look for certs in. None indicates to use + find_cert_dir() to find the path. Defaults to None. + Returns: + tuple or None: If no certificate is found, None is returned, otherwise, a 2-tuple + is returned, the first element is the path to a certificate, the second element + is the path to the matching key. + Raises: + RuntimeError: If a key is found without a matching certificate or vice versa. + """ + if cert_dir is None: + cert_dir = self._find_cert_dir() + + if cert_dir is None: + return None + + for l in sorted(os.listdir(cert_dir)): + if l.endswith('.cert'): + certpath = os.path.join(cert_dir, l) + keypath = certpath[:-5] + '.key' + if not os.path.exists(keypath): + raise RuntimeError("Cannot find key file for {}".format(certpath)) + return (certpath, keypath) + elif l.endswith('.key'): + # Should have found .cert first + keypath = os.path.join(cert_dir, l) + raise RuntimeError("Cannot find certificate file for {}".format(keypath)) + + return None + + def _wrap_method(self, f, relative_url, *args, **kwargs): + """ + Perform an HTTP request with appropriate options and fallback handling. + + This is used to implement methods like get, head, etc. It modifies + kwargs, tries to do the operation, then if a TLS request fails and + TLS validation is not required, tries again with a non-TLS URL. + + Args: + f (callable): callback to actually perform the request. + relative_url (str): URL relative to the toplevel hostname. + kwargs: Additional arguments passed to requests.Session.get. + Returns: + requests.Response: The response object. + """ + kwargs['auth'] = self.auth + kwargs['cert'] = self.cert + kwargs['verify'] = not self.insecure + res = None + if self._fallback: + try: + res = f(self._base + relative_url, *args, **kwargs) + self._fallback = None # don't fallback after one success + except (SSLError, ConnectionError): + self._base = self._fallback + self._fallback = None + if res is None: + res = f(self._base + relative_url, *args, **kwargs) + return res + + def get(self, relative_url, **kwargs): + """ + Do a HTTP GET. + + Args: + relative_url (str): URL relative to the toplevel hostname. + kwargs: Additional arguments passed to requests.Session.get. + Returns: + requests.Response: The response object. + """ + return self._wrap_method(self.session.get, relative_url, **kwargs) + + def head(self, relative_url, data=None, **kwargs): + """ + Do a HTTP HEAD. + + Args: + relative_url (str): URL relative to the toplevel hostname. + kwargs: Additional arguments passed to requests.Session.head. + Returns: + requests.Response: The response object. + """ + return self._wrap_method(self.session.head, relative_url, **kwargs) + + def post(self, relative_url, data=None, **kwargs): + """ + Do a HTTP POST. + + Args: + relative_url (str): URL relative to the toplevel hostname. + data: Data to include with the post, as for requests.SESSION. + kwargs: Additional arguments passed to requests.Session.post. + Returns: + requests.Response: The response object. + """ + return self._wrap_method(self.session.post, relative_url, data=data, **kwargs) + + def put(self, relative_url, data=None, **kwargs): + """ + Do a HTTP PUT. + + Args: + relative_url (str): URL relative to the toplevel hostname. + data: Data to include with the put, as for requests.SESSION. + kwargs: Additional arguments passed to requests.Session.put. + Returns: + requests.Response: The response object. + """ + return self._wrap_method(self.session.put, relative_url, data=data, **kwargs) + + +class ManifestInfo(object): + """Information about a manifest downloaded from the registry.""" + + def __init__(self, contents, digest, media_type, size): + """ + Initialize the ManifestInfo. + + Args: + contents (bytes): The contents. + media_type (str): The type of the content. + digest (str): The digest of the content. + size (int): The size of the download, in bytes. + """ + self.contents = contents + self.digest = digest + self.media_type = media_type + self.size = size + + +def get_manifest(session, repository, ref): + """ + Download a manifest from a registry. + + Args: + session (RegistrySession): The session object. + repository (str): The repository to download from. + ref (str): A digest, or a tag. + Returns: + ManifestInfo: Information about the downloaded content. + """ + logger.debug("%s: Retrieving manifest for %s:%s", session.registry, repository, ref) + + headers = { + 'Accept': ', '.join(( + MEDIA_TYPE_MANIFEST_V2, + MEDIA_TYPE_LIST_V2, + MEDIA_TYPE_OCI, + MEDIA_TYPE_OCI_INDEX + )) + } + + url = '/v2/{}/manifests/{}'.format(repository, ref) + response = session.get(url, headers=headers) + response.raise_for_status() + return ManifestInfo(response.content, + response.headers['Docker-Content-Digest'], + response.headers['Content-Type'], + int(response.headers['Content-Length'])) + + +class DirectoryEndpoint(object): + """ + The source or destination of a copy operation to a local directory. + + This is used only as a local intermediate, and for simplicity, the storage format is + not exactly the same as for a dir:// reference as understood by skopeo. + """ + + def __init__(self, directory): + """ + Initialize the DirectoryEndpoint. + + Args: + directory (str): The path to the directory. + """ + self.directory = directory + + def start_write(self): + """Do setup before writing to the endpoint.""" + with open(os.path.join(self.directory, 'oci-layout'), 'w') as f: + f.write('{"imageLayoutVersion": "1.0.0"}\n') + + def get_blob_path(self, digest): + """ + Get the path that a blob with the given digest would be stored at. + + Args: + digest (str): The digest of a blob to be stored. + Returns: + str: The full path where the blob would be stored. + """ + algorithm, digest = digest.split(':', 2) + return os.path.join(self.directory, 'blobs', algorithm, digest) + + def ensure_blob_path(self, digest): + """ + Get path for a blob, creating parent directories if necessary. + + Args: + digest (str): The digest of a blob to be stored. + Returns: + str: The full path where the blob would be stored. + """ + path = self.get_blob_path(digest) + + parent = os.path.dirname(path) + if not os.path.exists(parent): + os.makedirs(parent) + + return path + + def get_blob(self, digest): + """ + Return the contents of a blob object. + + Args: + digest (str): The digest to retrieve. + Returns: + bytes: The contents of the blob. + """ + with open(self.get_blob_path(digest), 'rb') as f: + return f.read() + + def has_blob(self, digest): + """ + Check if the repository has a blob with the given digest. + + Args: + digest (str): The digest to check for. + Returns: + bool: True if the blob exists. + """ + return os.path.exists(self.get_blob_path(digest)) + + def write_blob(self, digest, contents): + """ + Save a blob to the directory endpoint. + + Args: + digest (str): The digest of the blob's contents. + contents (bytes): The contents of the blob. + """ + path = self.ensure_blob_path(digest) + + with open(path, 'wb') as f: + f.write(contents) + + def get_manifest(self, digest=None, media_type=None): + """ + Get a manifest from the endpoint. + + Arguments: + digest (str): The digest of manifest to retrieve, or None to get the + main manifest for the endpoint. + media_type (str): The expected media type of the manifest. + Returns: + ManifestInfo: An object containing the contents of the manifest + and other relevant information. + """ + if digest is not None: + contents = self.get_blob(digest) + else: + manifest_path = os.path.join(self.directory, 'manifest.json') + if os.path.exists(manifest_path): + with open(manifest_path, 'rb') as f: + contents = f.read() + parsed = json.loads(contents) + media_type = parsed.get('mediaType', MEDIA_TYPE_OCI) + else: + index_path = os.path.join(self.directory, 'index.json') + with open(index_path, 'rb') as f: + contents = f.read() + parsed = json.loads(contents) + media_type = parsed.get('mediaType', MEDIA_TYPE_OCI_INDEX) + + return ManifestInfo(contents, digest, media_type, len(contents)) + + def write_manifest(self, info, toplevel=False): + """ + Store a manifest to the endpoint. + + Args: + info (ManifestInfo): An object containing the contents of the manifest + and other relevant information. + toplevel (bool): If True, this should be the main manifest stored + in the endpoint. + """ + if not toplevel: + self.write_blob(info.digest, info.contents) + elif info.media_type in (MEDIA_TYPE_LIST_V2, MEDIA_TYPE_OCI_INDEX): + with open(os.path.join(self.directory, 'index.json'), 'wb') as f: + f.write(info.contents) + else: + with open(os.path.join(self.directory, 'manifest.json'), 'wb') as f: + f.write(info.contents) + + +class RegistryEndpoint(object): + """The source or destination of a copy operation to a docker registry.""" + + def __init__(self, spec): + """ + Initialize the RegistryEndpoint. + + Args: + spec (RegistrySpec): A specification of registry, repository and tag. + """ + self.session = spec.get_session() + self.registry = spec.registry + self.repo = spec.repo + self.tag = spec.tag + + def start_write(self): + """Do setup before writing to the endpoint.""" + pass + + def download_blob(self, digest, size, blob_path): + """ + Download a blob from the registry to a local file. + + Args: + digest (str): The digest of the blob to download. + size (int): The size of blob. + blob_path (str): The local path to write the blob to. + """ + logger.info("%s: Downloading %s (size=%s)", self.registry, blob_path, size) + + url = "/v2/{}/blobs/{}".format(self.repo, digest) + result = self.session.get(url, stream=True) + result.raise_for_status() + + try: + with open(blob_path, 'wb') as f: + for block in result.iter_content(10 * 1024): + f.write(block) + finally: + result.close() + + def upload_blob(self, digest, size, blob_path): + """ + Upload a blob from a local file to the registry. + + Args: + digest (str): The digest of the blob to upload. + size (int): The size of blob to upload. + blob_path (str): The local path to read the blob from. + """ + logger.info("%s: Uploading %s (size=%s)", self.registry, blob_path, size) + + url = "/v2/{}/blobs/uploads/".format(self.repo) + result = self.session.post(url, data='') + result.raise_for_status() + + if result.status_code != requests.codes.ACCEPTED: + # if it was a failed response 4xx or 5xx then the raise_for_status() + # would have raised - so it's a "successful" response - but by the docker v2 api, + # a 202 ACCEPTED should be found here, not a 200 or other successful response. + raise RuntimeError("Unexpected successful response %s (202 expected)", + result.status_code) + + upload_url = result.headers.get('Location') + parsed = urlparse(upload_url) + if parsed.query == '': + query = 'digest=' + digest + else: + query = parsed.query + '&digest=' + digest + relative = urlunparse(('', '', parsed.path, parsed.params, query, '')) + + headers = { + 'Content-Length': str(size), + 'Content-Type': 'application/octet-stream' + } + with open(blob_path, 'rb') as f: + result = self.session.put(relative, data=f, headers=headers) + + result.raise_for_status() + if result.status_code != requests.codes.CREATED: + # if it was a failed response 4xx or 5xx then the raise_for_status() + # would have raised - so it's a "successful" response - but by the docker v2 api, + # a 202 CREATED should be found here, not a 200 or other successful response. + raise RuntimeError("Unexpected successful response %s (201 expected)", + result.status_code) + + def link_blob(self, digest, src_repo): + """ + Create a new reference to an object in another repository on the same registry. + + By using the "mount" operation from the docker protocol, we avoid having + to download and upload data. + + Args: + digest (str): The digest of the blob to create a new reference to. + src_repo (str): Another repository in the same registry which already has a + blob with the given digest. + """ + logger.info("%s: Linking blob %s from %s to %s", + self.registry, digest, src_repo, self.repo) + + # Check that it exists in the source repository + url = "/v2/{}/blobs/{}".format(src_repo, digest) + result = self.session.head(url) + result.raise_for_status() + + url = "/v2/{}/blobs/uploads/?mount={}&from={}".format(self.repo, digest, src_repo) + result = self.session.post(url, data='') + result.raise_for_status() + + if result.status_code != requests.codes.CREATED: + # A 202-Accepted would mean that the source blob didn't exist and + # we're starting an upload - but we've checked that above + raise RuntimeError("Blob mount had unexpected status {}".format(result.status_code)) + + def has_blob(self, digest): + """ + Check if the repository has a blob with the given digest. + + Args: + digest (str): The digest to check for. + Returns: + bool: True if the blob exists. + """ + url = "/v2/{}/blobs/{}".format(self.repo, digest) + result = self.session.head(url, stream=True) + if result.status_code == 404: + return False + result.raise_for_status() + return True + + def get_manifest(self, digest=None, media_type=None): + """ + Get a manifest from the endpoint. + + Args: + digest (str): The digest of manifest to retrieve, or None to get the + main manifest for the endpoint. + media_type (str): The expected media type of the manifest. + Returns: + ManifestInfo: An object containing the contents of the manifest + and other relevant information. + """ + if digest is None: + return get_manifest(self.session, self.repo, self.tag) + else: + return get_manifest(self.session, self.repo, digest) + + def write_manifest(self, info, toplevel=False): + """ + Store a manifest to the endpoint. + + Args: + info (ManifestInfo): An object containing the contents of the manifest + and other relevant information. + toplevel (bool): If True, this should be the main manifest stored + in the endpoint. + """ + if toplevel: + ref = self.tag + else: + ref = info.digest + + logger.info("%s: Storing manifest as %s", self.registry, ref) + + url = '/v2/{}/manifests/{}'.format(self.repo, ref) + headers = {'Content-Type': info.media_type} + response = self.session.put(url, data=info.contents, headers=headers) + response.raise_for_status() + + +class Copier(object): + """ + Implements a copy operation between different endpoints. + + As currently implemented, only copying to/from a registry and a directory + or within the *same* registry is implemented. + """ + + def __init__(self, src, dest): + """Initialize the Copier. + + Args: + src (str): The source endpoint. + dest (str): The destination endpoint. + """ + self.src = src + self.dest = dest + + def _copy_blob(self, digest, size): + """ + Copy a blob with given digest and size from the source to the destination. + + Args: + digest (str): The digest of the blob to copy. + size (int): The size of the blob to copy. + """ + if self.dest.has_blob(digest): + return + + if isinstance(self.src, RegistryEndpoint) and isinstance(self.dest, DirectoryEndpoint): + self.src.download_blob(digest, size, self.dest.ensure_blob_path(digest)) + elif isinstance(self.src, DirectoryEndpoint) and isinstance(self.dest, RegistryEndpoint): + self.dest.upload_blob(digest, size, self.src.get_blob_path(digest)) + elif isinstance(self.src, RegistryEndpoint) and isinstance(self.dest, RegistryEndpoint): + if self.src.registry == self.dest.registry: + self.dest.link_blob(digest, self.src.repo) + # Other forms of copying are not needed currently, and not implemented + + def _copy_manifest(self, info, toplevel=False): + """ + Copy the manifest referenced by info from the source to destination. + + Args: + info (ManifestInfo): References the manifest to be copied. + toplevel (bool): If True, this should be the main manifest referenced in the repository. + Defaults to False. + Raises: + RuntimeError: If the referenced media type is not supported by this client. + """ + references = [] + if info.media_type in (MEDIA_TYPE_MANIFEST_V2, MEDIA_TYPE_OCI): + manifest = json.loads(info.contents) + references.append((manifest['config']['digest'], manifest['config']['size'])) + for layer in manifest['layers']: + references.append((layer['digest'], layer['size'])) + else: + raise RuntimeError("Unhandled media type %s", info.media_type) + + for digest, size in references: + self._copy_blob(digest, size) + + self.dest.write_manifest(info, toplevel=toplevel) + + def copy(self): + """Perform the copy operation.""" + self.dest.start_write() + info = self.src.get_manifest() + if info.media_type in (MEDIA_TYPE_MANIFEST_V2, MEDIA_TYPE_OCI): + self._copy_manifest(info, toplevel=True) + elif info.media_type in (MEDIA_TYPE_LIST_V2, MEDIA_TYPE_OCI_INDEX): + manifest = json.loads(info.contents) + for m in manifest['manifests']: + referenced = self.src.get_manifest(digest=m['digest'], media_type=m['mediaType']) + self._copy_manifest(referenced) + self.dest.write_manifest(info, toplevel=True) + + else: + raise RuntimeError("Unhandled media type %s", info.media_type) + + +@main.command() +@click.option('--src-creds', '--screds', metavar='USERNAME[:PASSWORD]', + help='Use USERNAME[:PASSWORD] for accessing the source registry') +@click.option('--src-tls-verify', type=bool, default=True, + help=('require HTTPS and verify certificates when talking to the ' + 'container source registry (defaults to true)')) +@click.option('--src-cert-dir', metavar='PATH', + help=('use certificates at PATH (*.crt, *.cert, *.key) to connect to the ' + 'source registry')) +@click.option('--dest-creds', '--dcreds', metavar='USERNAME[:PASSWORD]', + help='Use USERNAME[:PASSWORD] for accessing the destination registry') +@click.option('--dest-tls-verify', type=bool, default=True, + help=('require HTTPS and verify certificates when talking to the ' + 'container destination registry (defaults to true)')) +@click.option('--dest-cert-dir', metavar='PATH', + help=('use certificates at PATH (*.crt, *.cert, *.key) to connect to the ' + 'destination registry')) +@click.argument('src', metavar='SOURCE-IMAGE-NAME') +@click.argument('dest', metavar='DEST-IMAGE-NAME') +def copy(src, dest, screds, src_tls_verify, src_cert_dir, dcreds, dest_tls_verify, dest_cert_dir): + """Copy an image from one location to another.""" + src = parse_spec(src, screds, src_tls_verify, src_cert_dir) + dest = parse_spec(dest, dcreds, dest_tls_verify, dest_cert_dir) + + if src.registry != dest.registry: + tempdir = tempfile.mkdtemp() + try: + tmp = DirectoryEndpoint(tempdir) + Copier(src.get_endpoint(), tmp).copy() + Copier(tmp, dest.get_endpoint()).copy() + finally: + shutil.rmtree(tempdir) + else: + Copier(src.get_endpoint(), dest.get_endpoint()).copy() + + +if __name__ == '__main__': + main() diff --git a/bodhi/tests/server/scripts/test_skopeo_lite.py b/bodhi/tests/server/scripts/test_skopeo_lite.py new file mode 100644 index 0000000000..f374489051 --- /dev/null +++ b/bodhi/tests/server/scripts/test_skopeo_lite.py @@ -0,0 +1,732 @@ +# -*- coding: utf-8 -*- +# Copyright © 2018 Red Hat, Inc. +# +# This file is part of Bodhi. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +"""This module contains tests for bodhi.server.scripts.skopeo_lite.""" + +from base64 import b64encode +from contextlib import contextmanager +import hashlib +import json +import os +import re +import shutil +import tempfile +import uuid + +from click import testing +import mock +import pytest +import responses +import requests +from six import binary_type, text_type +from six.moves.urllib.parse import urlparse + +from bodhi.server.scripts import skopeo_lite +from bodhi.server.scripts.skopeo_lite import (MEDIA_TYPE_MANIFEST_V2, MEDIA_TYPE_LIST_V2, + MEDIA_TYPE_OCI, MEDIA_TYPE_OCI_INDEX) + + +REGISTRY_V1 = 'registry_v1.example.com' +REGISTRY_V2 = 'registry_v2.example.com' +OTHER_V2 = 'registry.example.com:5001' + +all_registry_conf = { + REGISTRY_V2: {'version': 'v2', 'insecure': True}, + OTHER_V2: {'version': 'v2', 'insecure': False}, +} + + +def registry_hostname(registry): + """ + Strip a reference to a registry to just the hostname:port + """ + if registry.startswith('http:') or registry.startswith('https:'): + return urlparse(registry).netloc + else: + return registry + + +def to_bytes(value): + if isinstance(value, binary_type): + return value + else: + return value.encode('utf-8') + + +def to_text(value): + if isinstance(value, text_type): + return value + else: + return text_type(value, 'utf-8') + + +def make_digest(blob): + # Abbreviate the hexdigest for readability of debugging output if things fail + return 'sha256:' + hashlib.sha256(to_bytes(blob)).hexdigest()[0:10] + + +class MockRegistry(object): + """ + This class mocks a subset of the v2 Docker Registry protocol. It also has methods to inject + and test content in the registry. + """ + def __init__(self, registry, insecure=False, required_creds=None, flags=''): + self.hostname = registry_hostname(registry) + self.insecure = insecure + self.repos = {} + self.required_creds = required_creds + self.flags = flags + self._add_pattern(responses.GET, r'/v2/(.*)/manifests/([^/]+)', + self._get_manifest) + self._add_pattern(responses.HEAD, r'/v2/(.*)/manifests/([^/]+)', + self._get_manifest) + self._add_pattern(responses.PUT, r'/v2/(.*)/manifests/([^/]+)', + self._put_manifest) + self._add_pattern(responses.GET, r'/v2/(.*)/blobs/([^/]+)', + self._get_blob) + self._add_pattern(responses.HEAD, r'/v2/(.*)/blobs/([^/]+)', + self._get_blob) + self._add_pattern(responses.POST, r'/v2/(.*)/blobs/uploads/', + self._post_blob) + self._add_pattern(responses.PUT, r'/v2/(.*)/blobs/uploads/([^?]*)\?digest=(.*)', + self._put_blob) + self._add_pattern(responses.PUT, r'/v2/(.*)/blobs/uploads/([^?]*)\?dummy=1&digest=(.*)', + self._put_blob) + self._add_pattern(responses.POST, r'/v2/(.*)/blobs/uploads/\?mount=([^&]+)&from=(.+)', + self._mount_blob) + + def get_repo(self, name): + return self.repos.setdefault(name, { + 'blobs': {}, + 'manifests': {}, + 'tags': {}, + 'uploads': {}, + }) + + def add_blob(self, name, blob): + repo = self.get_repo(name) + digest = make_digest(blob) + repo['blobs'][digest] = blob + return digest + + def get_blob(self, name, digest): + return self.get_repo(name)['blobs'][digest] + + def add_manifest(self, name, ref, manifest): + repo = self.get_repo(name) + digest = make_digest(manifest) + repo['manifests'][digest] = manifest + if ref is None: + pass + elif ref.startswith('sha256:'): + assert ref == digest + else: + repo['tags'][ref] = digest + return digest + + def get_manifest(self, name, ref): + repo = self.get_repo(name) + if not ref.startswith('sha256:'): + ref = repo['tags'][ref] + return repo['manifests'][ref] + + def _check_creds(self, req): + if self.required_creds: + username, password = self.required_creds + + auth = req.headers['Authorization'].strip().split() + assert auth[0] == 'Basic' + assert to_bytes(auth[1]) == b64encode(to_bytes(username + ':' + password)) + + def _add_pattern(self, method, pattern, callback): + if self.insecure: + url = 'http://' + self.hostname + else: + url = 'https://' + self.hostname + pat = re.compile('^' + url + pattern + '$') + + def do_it(req): + self._check_creds(req) + + status, headers, body = callback(req, *(pat.match(req.url).groups())) + if method == responses.HEAD: + return status, headers, '' + else: + return status, headers, body + + responses.add_callback(method, pat, do_it, match_querystring=True) + + def _get_manifest(self, req, name, ref): + repo = self.get_repo(name) + if not ref.startswith('sha256:'): + try: + ref = repo['tags'][ref] + except KeyError: + return (requests.codes.NOT_FOUND, {}, {'error': 'NOT_FOUND'}) + + try: + blob = repo['manifests'][ref] + except KeyError: + return (requests.codes.NOT_FOUND, {}, {'error': 'NOT_FOUND'}) + + decoded = json.loads(to_text(blob)) + content_type = decoded.get('mediaType') + if content_type is None: # OCI + if decoded.get('manifests') is not None: + content_type = MEDIA_TYPE_OCI_INDEX + else: + content_type = MEDIA_TYPE_OCI + + accepts = re.split(r'\s*,\s*', req.headers['Accept']) + assert content_type in accepts + + if 'bad_index_content_type' in self.flags: + if content_type == MEDIA_TYPE_OCI_INDEX: + content_type = 'application/json' + if 'bad_content_type' in self.flags: + if content_type == MEDIA_TYPE_OCI: + content_type = 'application/json' + + headers = { + 'Docker-Content-Digest': ref, + 'Content-Type': content_type, + 'Content-Length': str(len(blob)), + } + return (200, headers, blob) + + def _put_manifest(self, req, name, ref): + try: + json.loads(to_text(req.body)) + except ValueError: + return (400, {}, {'error': 'BAD_MANIFEST'}) + + self.add_manifest(name, ref, req.body) + return (200, {}, '') + + def _get_blob(self, req, name, digest): + repo = self.get_repo(name) + assert digest.startswith('sha256:') + + try: + blob = repo['blobs'][digest] + except KeyError: + return (requests.codes.NOT_FOUND, {}, {'error': 'NOT_FOUND'}) + + headers = { + 'Docker-Content-Digest': digest, + 'Content-Type': 'application/json', + 'Content-Length': str(len(blob)), + } + return (200, headers, blob) + + def _post_blob(self, req, name): + repo = self.get_repo(name) + uuid_str = str(uuid.uuid4()) + repo['uploads'][uuid_str] = '' + + if 'include_query_parameters' in self.flags: + location = '/v2/{}/blobs/uploads/{}?dummy=1'.format(name, uuid_str) + else: + location = '/v2/{}/blobs/uploads/{}'.format(name, uuid_str) + + headers = { + 'Location': location, + 'Range': 'bytes=0-0', + 'Content-Length': '0', + 'Docker-Upload-UUID': uuid_str, + } + return (200 if 'bad_post_status' in self.flags else 202, headers, '') + + def _put_blob(self, req, name, uuid, digest): + repo = self.get_repo(name) + + assert uuid in repo['uploads'] + del repo['uploads'][uuid] + + if isinstance(req.body, binary_type) or isinstance(req.body, text_type): + blob = req.body + else: + blob = req.body.read() + + added_digest = self.add_blob(name, blob) + assert added_digest == digest + + headers = { + 'Location': '/v2/{}/blobs/{}'.format(name, digest), + 'Docker-Content-Digest': added_digest, + } + + return (200 if 'bad_put_status' in self.flags else 201, headers, '') + + def _mount_blob(self, req, target_name, digest, source_name): + source_repo = self.get_repo(source_name) + target_repo = self.get_repo(target_name) + + try: + target_repo['blobs'][digest] = source_repo['blobs'][digest] + headers = { + 'Location': '/v2/{}/blobs/{}'.format(target_name, digest), + 'Docker-Content-Digest': digest, + } + return (200 if 'bad_mount_status' in self.flags else 201, headers, '') + except KeyError: + headers = { + 'Location': '/v2/{}/blobs/uploads/some-uuid'.format(target_name), + 'Docker-Upload-UUID': 'some-uuid', + } + return (202, headers, '') + + def add_fake_image(self, name, tag, content_type, + arch='amd64'): + layer_digest = self.add_blob(name, 'layer-' + arch) + layer_size = len(to_bytes('layer-' + arch)) + + config = { + 'architecture': arch, + 'os': 'linux', + } + config_bytes = to_bytes(json.dumps(config)) + config_digest = self.add_blob(name, config_bytes) + config_size = len(config_bytes) + + if content_type in (MEDIA_TYPE_MANIFEST_V2, MEDIA_TYPE_LIST_V2): + manifest = { + 'schemaVersion': 2, + 'mediaType': MEDIA_TYPE_MANIFEST_V2, + 'config': { + 'mediaType': 'application/vnd.docker.container.image.v1+json', + 'digest': config_digest, + 'size': config_size, + }, + 'layers': [{ + 'mediaType': 'application/vnd.docker.image.rootfs.diff.tar.gzip', + 'digest': layer_digest, + 'size': layer_size, + }] + } + + if content_type == MEDIA_TYPE_LIST_V2: + manifest_bytes = to_bytes(json.dumps(manifest)) + manifest_digest = self.add_manifest(name, None, manifest_bytes) + + manifest = { + 'schemaVersion': 2, + 'mediaType': MEDIA_TYPE_LIST_V2, + 'manifests': [{ + 'mediaType': MEDIA_TYPE_MANIFEST_V2, + 'size': len(manifest_bytes), + 'digest': manifest_digest, + 'platform': { + 'architecture': arch, + 'os': 'linux', + } + }] + } + else: + manifest = { + 'schemaVersion': 2, + 'mediaType': MEDIA_TYPE_OCI, + 'config': { + 'mediaType': 'application/vnd.oci.image.config.v1+json', + 'digest': config_digest, + 'size': config_size, + }, + 'layers': [{ + 'mediaType': 'application/vnd.oci.image.layer.v1.tar', + 'digest': layer_digest, + 'size': layer_size, + }] + } + + if content_type == MEDIA_TYPE_OCI_INDEX: + manifest_bytes = to_bytes(json.dumps(manifest)) + manifest_digest = self.add_manifest(name, None, manifest_bytes) + + manifest = { + 'schemaVersion': 2, + 'manifests': [{ + 'mediaType': MEDIA_TYPE_OCI, + 'size': len(manifest_bytes), + 'digest': manifest_digest, + 'platform': { + 'architecture': arch, + 'os': 'linux', + } + }] + } + + manifest_bytes = to_bytes(json.dumps(manifest)) + return self.add_manifest(name, tag, manifest_bytes) + + def check_fake_image(self, name, tag, digest, content_type): + manifest_bytes = self.get_manifest(name, tag) + assert make_digest(manifest_bytes) == digest + + manifest = json.loads(to_text(manifest_bytes)) + if content_type in (MEDIA_TYPE_LIST_V2, MEDIA_TYPE_OCI_INDEX): + manifest_digest = manifest['manifests'][0]['digest'] + manifest_bytes = self.get_manifest(name, manifest_digest) + manifest = json.loads(to_text(manifest_bytes)) + + config_digest = manifest['config']['digest'] + assert make_digest(self.get_blob(name, config_digest)) == config_digest + + layer_digest = manifest['layers'][0]['digest'] + assert make_digest(self.get_blob(name, layer_digest)) == layer_digest + + +@responses.activate +@pytest.mark.parametrize('content_type', + (MEDIA_TYPE_OCI, MEDIA_TYPE_OCI_INDEX, + MEDIA_TYPE_MANIFEST_V2, MEDIA_TYPE_LIST_V2)) +def test_skopeo_copy_basic(content_type): + """ + Test copying from one server to another + """ + runner = testing.CliRunner() + + reg1 = MockRegistry('registry1.example.com') + reg2 = MockRegistry('registry2.example.com') + digest = reg1.add_fake_image('repo1', 'latest', content_type) + + result = runner.invoke( + skopeo_lite.copy, + ['docker://registry1.example.com/repo1:latest', + 'docker://registry2.example.com/repo2:latest'], + catch_exceptions=False) + + assert result.exit_code == 0 + + reg2.check_fake_image('repo2', 'latest', digest, content_type) + + +@responses.activate +@pytest.mark.parametrize('content_type', + (MEDIA_TYPE_OCI, MEDIA_TYPE_OCI_INDEX, + MEDIA_TYPE_MANIFEST_V2, MEDIA_TYPE_LIST_V2)) +def test_skopeo_copy_link(content_type): + """ + Testing copying on the same server, avoiding download/upload + """ + runner = testing.CliRunner() + + reg1 = MockRegistry('registry1.example.com') + digest = reg1.add_fake_image('repo1', 'latest', content_type) + + result = runner.invoke( + skopeo_lite.copy, + ['docker://registry1.example.com/repo1:latest', + 'docker://registry1.example.com/repo2:latest'], + catch_exceptions=False) + + assert result.exit_code == 0 + + reg1.check_fake_image('repo2', 'latest', digest, content_type) + + +@responses.activate +@pytest.mark.parametrize('content_type', + (MEDIA_TYPE_OCI, MEDIA_TYPE_OCI_INDEX, + MEDIA_TYPE_MANIFEST_V2, MEDIA_TYPE_LIST_V2)) +def test_skopeo_copy_tag(content_type): + """ + Testing copying within the same repo - creating a new tag for an existing image + """ + runner = testing.CliRunner() + + reg1 = MockRegistry('registry1.example.com') + digest = reg1.add_fake_image('repo1', '1.2.3', content_type) + + # No tag should be the same as :latest + result = runner.invoke( + skopeo_lite.copy, + ['docker://registry1.example.com/repo1:1.2.3', 'docker://registry1.example.com/repo1'], + catch_exceptions=False) + + assert result.exit_code == 0 + + reg1.check_fake_image('repo1', 'latest', digest, content_type) + + +@responses.activate +@pytest.mark.parametrize('insecure', (True, False)) +def test_skopeo_copy_insecure(insecure): + """ + Testing falling back to HTTP when talking to a server + """ + runner = testing.CliRunner() + + content_type = MEDIA_TYPE_MANIFEST_V2 + reg1 = MockRegistry('registry1.example.com', insecure=insecure) + digest = reg1.add_fake_image('repo1', '1.2.3', content_type) + + result = runner.invoke( + skopeo_lite.copy, + ['--src-tls-verify', 'false', + '--dest-tls-verify', 'false', + 'docker://registry1.example.com/repo1:1.2.3', + 'docker://registry1.example.com/repo1:latest'], + catch_exceptions=False) + + assert result.exit_code == 0 + + reg1.check_fake_image('repo1', 'latest', digest, content_type) + + +@responses.activate +def test_skopeo_copy_username_password(): + """ + Testing authentication with username and password + """ + runner = testing.CliRunner() + + content_type = MEDIA_TYPE_MANIFEST_V2 + reg1 = MockRegistry('registry1.example.com', required_creds=('someuser', 'somepassword')) + reg2 = MockRegistry('registry2.example.com', required_creds=('otheruser', 'otherpassword')) + digest = reg1.add_fake_image('repo1', 'latest', content_type) + + result = runner.invoke( + skopeo_lite.copy, + ['--src-creds', 'someuser:somepassword', + '--dest-creds', 'otheruser:otherpassword', + 'docker://registry1.example.com/repo1:latest', + 'docker://registry2.example.com/repo2:latest'], + catch_exceptions=False) + + assert result.exit_code == 0 + + reg2.check_fake_image('repo2', 'latest', digest, content_type) + + +@contextmanager +def check_certificates(get_cert=None, put_cert=None): + old_get = requests.Session.get + old_put = requests.Session.put + + def checked_get(self, *args, **kwargs): + if kwargs.get('cert') != get_cert: + raise RuntimeError("Wrong/missing cert for GET") + + return old_get(self, *args, **kwargs) + + def checked_put(self, *args, **kwargs): + if kwargs.get('cert') != put_cert: + raise RuntimeError("Wrong/missing cert for PUT") + + return old_put(self, *args, **kwargs) + + with mock.patch('requests.Session.get', autospec=True, side_effect=checked_get): + with mock.patch('requests.Session.put', autospec=True, side_effect=checked_put): + yield + + +@responses.activate +@pytest.mark.parametrize(('breakage', 'error'), [ + (None, None), + ('missing_cert', 'Cannot find certificate file'), + ('missing_key', 'Cannot find key file'), + ('missing_cert_and_key', 'Wrong/missing cert'), +]) +def test_skopeo_copy_cert(breakage, error): + """ + Test authentication with a certificate + """ + runner = testing.CliRunner() + + certdir1 = tempfile.mkdtemp() + certdir2 = tempfile.mkdtemp() + try: + certs = {} + for certdir, reg in ((certdir1, 'registry1.example.com'), + (certdir2, 'registry2.example.com')): + cert = os.path.join(certdir, reg + '.cert') + if breakage not in ('missing_cert', 'missing_cert_and_key'): + with open(cert, 'w'): + pass + key = os.path.join(certdir, reg + '.key') + if breakage not in ('missing_key', 'missing_cert_and_key'): + with open(key, 'w'): + pass + certs[reg] = (cert, key) + + content_type = MEDIA_TYPE_MANIFEST_V2 + reg1 = MockRegistry('registry1.example.com') + reg2 = MockRegistry('registry2.example.com') + digest = reg1.add_fake_image('repo1', 'latest', content_type) + + with check_certificates(get_cert=certs['registry1.example.com'], + put_cert=certs['registry2.example.com']): + args = ['--src-cert-dir', certdir1, + '--dest-cert-dir', certdir2, + 'docker://registry1.example.com/repo1:latest', + 'docker://registry2.example.com/repo2:latest'] + + if breakage is None: + result = runner.invoke( + skopeo_lite.copy, + args, + catch_exceptions=False) + + assert result.exit_code == 0 + reg2.check_fake_image('repo2', 'latest', digest, content_type) + else: + with pytest.raises(Exception) as e: + runner.invoke( + skopeo_lite.copy, + args, + catch_exceptions=False) + assert error in str(e) + finally: + shutil.rmtree(certdir1) + shutil.rmtree(certdir2) + + +@contextmanager +def mock_system_certs(): + old_isdir = os.path.isdir + old_listdir = os.listdir + old_exists = os.path.exists + + def isdir(path): + if isinstance(path, text_type) and path.startswith('/etc/'): + return path in ('/etc/docker/certs.d/registry1.example.com', + '/etc/docker/certs.d/registry2.example.com') + else: + return old_isdir(path) + + def listdir(path): + if isinstance(path, text_type) and path.startswith('/etc/'): + if path in ('/etc/docker/certs.d/registry1.example.com', + '/etc/docker/certs.d/registry2.example.com'): + return ('client.cert', 'client.key') + else: + return None + else: + return old_listdir(path) + + def exists(path): + if isinstance(path, text_type) and path.startswith('/etc/'): + return path in ('/etc/docker/certs.d/registry1.example.com/client.cert', + '/etc/docker/certs.d/registry1.example.com/client.key', + '/etc/docker/certs.d/registry2.example.com/client.cert', + '/etc/docker/certs.d/registry2.example.com/client.key') + else: + return old_exists(path) + + with mock.patch('os.path.isdir', side_effect=isdir): + with mock.patch('os.listdir', side_effect=listdir): + with mock.patch('os.path.exists', side_effect=exists): + yield + + +@responses.activate +def test_skopeo_copy_system_cert(): + """ + Test using a certificate from a system directory + """ + runner = testing.CliRunner() + + reg1 = MockRegistry('registry1.example.com') + MockRegistry('registry2.example.com') + reg1.add_fake_image('repo1', 'latest', MEDIA_TYPE_MANIFEST_V2) + + with mock_system_certs(): + with check_certificates(get_cert=('/etc/docker/certs.d/registry1.example.com/client.cert', + '/etc/docker/certs.d/registry1.example.com/client.key'), + put_cert=('/etc/docker/certs.d/registry2.example.com/client.cert', + '/etc/docker/certs.d/registry2.example.com/client.key')): + runner.invoke( + skopeo_lite.copy, + ['docker://registry1.example.com/repo1:latest', + 'docker://registry2.example.com/repo2:latest'], + catch_exceptions=False) + + +@responses.activate +@pytest.mark.parametrize(('src', 'dest', 'error'), [ + ('docker://registry1.example.com/repo1:latest', 'badtype://registry2.example.com/repo2:latest', + 'Unknown source/destination'), + ('docker://registry1.example.com/repo1:latest', 'docker:registry1.example.com/repo2:latest', + 'Registry specification should be docker://REGISTRY/PATH'), + ('docker://registry1.example.com/repo1:latest', 'docker://registry2.example.com', + 'Registry specification should be docker://REGISTRY/PATH'), +]) +def test_skopeo_copy_cli_errors(src, dest, error): + """ + Test errors triggered from bad command line arguments + """ + runner = testing.CliRunner() + + options = [] + + result = runner.invoke( + skopeo_lite.copy, + options + [src, dest]) + + assert error in result.output + assert result.exit_code != 0 + + +@responses.activate +@pytest.mark.parametrize(('src', 'dest', 'flags1', 'flags2', 'error'), [ + ('docker://registry1.example.com/repo1:latest', 'docker://registry2.example.com/repo2:latest', + '', 'bad_put_status', + 'Unexpected successful response'), + ('docker://registry1.example.com/repo1:latest', 'docker://registry2.example.com/repo2:latest', + '', 'bad_post_status', + 'Unexpected successful response'), + ('docker://registry1.example.com/repo1:latest', 'docker://registry1.example.com/repo2:latest', + 'bad_mount_status', '', + 'Blob mount had unexpected status'), + ('docker://registry1.example.com/repo1:latest', 'docker://registry2.example.com/repo2:latest', + 'bad_index_content_type', '', + 'Unhandled media type'), + ('docker://registry1.example.com/repo1:latest', 'docker://registry2.example.com/repo2:latest', + 'bad_content_type', '', + 'Unhandled media type'), + ('docker://registry1.example.com/repo1:latest', 'docker://registry2.example.com/repo2:latest', + '', 'include_query_parameters', + None), +]) +def test_skopeo_copy_protocol(src, dest, flags1, flags2, error): + """ + Tests various error and other code paths related to variations in server responses; the + flags argument to the MockRegistry constructor is used to modify server behavior. + """ + runner = testing.CliRunner() + + content_type = MEDIA_TYPE_OCI_INDEX + + reg1 = MockRegistry('registry1.example.com', flags=flags1) + MockRegistry('registry2.example.com', flags=flags2) + reg1.add_fake_image('repo1', 'latest', content_type) + + if error is not None: + with pytest.raises(Exception) as e: + runner.invoke( + skopeo_lite.copy, + [src, dest], + catch_exceptions=False) + + assert error in str(e) + else: + result = runner.invoke( + skopeo_lite.copy, + [src, dest]) + + assert result.exit_code == 0 diff --git a/devel/ansible/roles/dev/tasks/main.yml b/devel/ansible/roles/dev/tasks/main.yml index 2317696847..7da123e950 100644 --- a/devel/ansible/roles/dev/tasks/main.yml +++ b/devel/ansible/roles/dev/tasks/main.yml @@ -57,6 +57,7 @@ - python2-pyramid-mako - python2-pyramid-tm - python2-pytest-cov + - python2-responses - python2-rpdb - python2-simplemediawiki - python2-sphinx @@ -89,6 +90,7 @@ - python3-pyramid-fas-openid - python3-pytest - python3-pytest-cov + - python3-responses - python3-simplemediawiki - python3-sqlalchemy - python3-webtest diff --git a/devel/ci/pip-packages b/devel/ci/pip-packages index 1f7b78e293..47756e273b 100644 --- a/devel/ci/pip-packages +++ b/devel/ci/pip-packages @@ -15,6 +15,7 @@ RUN pip-2 install \ diff-cover \ flake8 \ mock \ + responses \ pytest \ pytest-cov \ sqlalchemy_schemadisplay \ @@ -22,6 +23,7 @@ RUN pip-2 install \ RUN pip-3 install \ diff-cover \ mock \ + responses \ pydocstyle \ pytest \ pytest-cov \ diff --git a/devel/ci/rpm-packages b/devel/ci/rpm-packages index f811906a89..7f742367a3 100644 --- a/devel/ci/rpm-packages +++ b/devel/ci/rpm-packages @@ -14,6 +14,7 @@ python2-pyramid-mako \ python2-pyramid-tm \ python2-pytest-cov \ + python2-responses \ python2-sqlalchemy \ python2-sqlalchemy_schemadisplay \ python3-alembic \ @@ -36,6 +37,7 @@ python3-pyramid-tm \ python3-pytest \ python3-pytest-cov \ + python3-responses \ python3-simplemediawiki \ python3-sqlalchemy \ python3-webtest \ diff --git a/docs/conf.py b/docs/conf.py index 02b9899ed0..2ef47e4bf4 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -263,6 +263,9 @@ ('user/man_pages/bodhi-expire-overrides', 'bodhi-expire-overrides', 'Look for overrides that are past their expiration dates and mark them expired', ['Randy Barlow'], 1), + ('user/man_pages/bodhi-skopeo-lite', 'bodhi-skopeo-lite', + 'Copy containers between registries', + ['Owen Taylor'], 1), ('user/man_pages/bodhi-untag-branched', 'bodhi-untag-branched', 'Remove the pending and testing tags from updates in a branched release.', ['Randy Barlow'], 1), diff --git a/docs/user/man_pages/bodhi-skopeo-lite.rst b/docs/user/man_pages/bodhi-skopeo-lite.rst new file mode 100644 index 0000000000..e280564f0d --- /dev/null +++ b/docs/user/man_pages/bodhi-skopeo-lite.rst @@ -0,0 +1,79 @@ +================= +bodhi-skopeo-lite +================= + +Synopsis +======== + +``bodhi-skopeo-lite`` COMMAND [OPTIONS] [ARGS]... + + +Description +=========== + +``bodhi-skopeo-lite`` is a very limited version of the `skopeo `_ +tool, but with support for manifests lists and OCI image indexes. The only command that is supported is +``copy``, and the only supported image references are Docker registry references of the form +``docker://docker-reference``. + + + +Options +======= + +``--help`` + + Show help text and exit. + + +Commands +======== + +There is one command, ``copy``. + +``bodhi-skopeo-lite copy [options] source-image destination-image`` + +The ``copy`` command copies an image from one location to another. It supports +the following options: + +``--src-creds, --screds [:]`` + + Use ``username`` and ``password`` for accessing the source registry. + +``-src-tls-verify `` + + Require HTTPS and verify certificates when talking to the container + source registry (defaults to ``true``). + +``--src-cert-dir `` + + Use certificates at ``path`` (\*.crt, \*.cert, \*.key) to connect to the source registry. + +``-dest-creds, --dcreds [:]`` + + Use ``username`` and ``password`` for accessing the destination registry. + +``--dest-tls-verify `` + + Require HTTPS and verify certificates when talking to the container + destination registry (defaults to ``true``). + +``--dest-cert-dir `` + + Use certificates at ``path`` (\*.crt, \*.cert, \*.key) to connect to the destination + registry. + +``--help`` + + Show help text and exit. + + +Help +==== + +If you find bugs in bodhi (or in the man page), please feel free to file a bug report or a pull +request:: + + https://github.com/fedora-infra/bodhi + +Bodhi's documentation is available online: https://bodhi.fedoraproject.org/docs diff --git a/docs/user/man_pages/index.rst b/docs/user/man_pages/index.rst index ad54883b80..25c6fc0a7e 100644 --- a/docs/user/man_pages/index.rst +++ b/docs/user/man_pages/index.rst @@ -14,5 +14,6 @@ Man pages bodhi-manage-releases bodhi-monitor-composes bodhi-push + bodhi-skopeo-lite bodhi-untag-branched initialize_bodhi_db diff --git a/docs/user/release_notes.rst b/docs/user/release_notes.rst index 922f2758c1..e2fc32c7ea 100644 --- a/docs/user/release_notes.rst +++ b/docs/user/release_notes.rst @@ -26,6 +26,7 @@ Dependency changes * Cornice must now be at least version 3.1.0 (:issue:`2286`). * Greenwave is now a required service for Bodhi deployments that wish to continue displaying test results in the UI (:issue:`2370`). +* The responses python module is now needed for running tests. Features diff --git a/production.ini b/production.ini index f1782caf34..af33572d32 100644 --- a/production.ini +++ b/production.ini @@ -173,6 +173,13 @@ use = egg:bodhi-server # The skopeo executable to use to copy container images. # You can put credentials for skopeo to use in $HOME/.docker/config.json # https://github.com/projectatomic/skopeo#private-registries-with-authentication +# +# An alternative command is bodhi-skopeo-lite, installed by bodhi. It has various limitations +# (including not reading $HOME/.docker/config.json - it supports certificate authentication +# and credentials passed on the command line), but supports manifest lists and OCI image indexes, +# allowing copying multi-arch containers. See https://github.com/containers/image/pull/400 for +# work to add such support to skopeo proper. +# # skopeo.cmd = /usr/bin/skopeo # Comma separated list of extra flags to pass to the skopeo copy command. diff --git a/setup.py b/setup.py index 0a2fc39fc1..55fcaa9573 100644 --- a/setup.py +++ b/setup.py @@ -162,6 +162,7 @@ def get_requirements(requirements_file='requirements.txt'): bodhi-approve-testing = bodhi.server.scripts.approve_testing:main bodhi-manage-releases = bodhi.server.scripts.manage_releases:main bodhi-check-policies = bodhi.server.scripts.check_policies:check + bodhi-skopeo-lite = bodhi.server.scripts.skopeo_lite:main [moksha.consumer] masher = bodhi.server.consumers.masher:Masher updates = bodhi.server.consumers.updates:UpdatesHandler