From e2ae6dbe72b6027b9e197a9a3c0f4e8b0f093ba5 Mon Sep 17 00:00:00 2001 From: "Owen W. Taylor" Date: Fri, 8 Jun 2018 12:00:09 -0400 Subject: [PATCH] Add bodhi-skopeo-lite - a skopeo-workalike with manifest list support In order to support copying multi-arch containers and Flatpaks, we need to be able to copy manifest lists and OCI image indexes from registry to registry. Work is underway to add such support to skopeo (https://github.com/containers/image/pull/400), but as a temporary workaround add 'bodhi-skopeo-lite', which implements the subset of 'skopeo copy' we need, but with manifest list/image index support. Use of this needs to be specifically configured. Signed-off-by: Owen W. Taylor --- .coveragerc | 1 + bodhi/server/scripts/skopeo_lite.py | 472 ++++++++++++ .../tests/server/scripts/test_skopeo_lite.py | 727 ++++++++++++++++++ devel/ci/pip-packages | 2 + devel/ci/rpm-packages | 2 + production.ini | 7 + setup.py | 1 + 7 files changed, 1212 insertions(+) create mode 100644 bodhi/server/scripts/skopeo_lite.py create mode 100644 bodhi/tests/server/scripts/test_skopeo_lite.py diff --git a/.coveragerc b/.coveragerc index 494941ba9d..5734597702 100644 --- a/.coveragerc +++ b/.coveragerc @@ -15,3 +15,4 @@ exclude_lines = def __repr__ def multicall_enabled if __name__ == .__main__.: + raise AssertionError diff --git a/bodhi/server/scripts/skopeo_lite.py b/bodhi/server/scripts/skopeo_lite.py new file mode 100644 index 0000000000..dcc216bb93 --- /dev/null +++ b/bodhi/server/scripts/skopeo_lite.py @@ -0,0 +1,472 @@ +# -*- coding: utf-8 -*- +# Copyright © 2018 Red Hat, Inc. and others. +# +# This file is part of Bodhi. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +"""Copy containers between registries. + +This is a very limited version of the skopeo tool, but with support +for manifests lists and OCI image indexes. +https://github.com/containers/image/pull/400 will make this +unnecessary. + +The only subcommand that is supported is 'copy', and the only supported image references +are Docker registries of the form 'docker://docker-reference'. + +No global options are supported, and only selected options to 'copy' are supported (see +--help for details.) + +Some other things that aren't implemented (but could be added if necessary): + - Handling of www-authenticate responses, necessary to log in to docker.io + - Special handling of 'docker.io' as 'registry-1.docker.io' + - Reading ~/.docker/config.json or $XDG_RUNTIME_DIR/containers/auth.json + - Handling foreign layers + +""" + +import click +import json +import logging +import os +import requests +from requests.exceptions import SSLError, ConnectionError +import shutil +import tempfile +from six.moves.urllib.parse import urlparse, urlunparse + + +@click.group() +def main(): + """Simplified Skopeo work-alike with manifest list support.""" + pass # pragma: no cover + + +logger = logging.getLogger('skopeo-lite') +logging.basicConfig(level=logging.INFO) + + +MEDIA_TYPE_MANIFEST_V2 = 'application/vnd.docker.distribution.manifest.v2+json' +MEDIA_TYPE_LIST_V2 = 'application/vnd.docker.distribution.manifest.list.v2+json' +MEDIA_TYPE_OCI = 'application/vnd.oci.image.manifest.v1+json' +MEDIA_TYPE_OCI_INDEX = 'application/vnd.oci.image.index.v1+json' + + +class RegistrySpec(object): + def __init__(self, registry, repo, tag, creds, tls_verify, cert_dir): + self.registry = registry + self.repo = repo + self.tag = tag + self.creds = creds + self.cert_dir = cert_dir + self.tls_verify = tls_verify + + def get_session(self): + return RegistrySession(self.registry, insecure=not self.tls_verify, + creds=self.creds, cert_dir=self.cert_dir) + + def get_endpoint(self): + return RegistryEndpoint(self) + + +def parse_spec(spec, creds, tls_verify, cert_dir): + if spec.startswith('docker:'): + _, rest = spec.split(':', 1) + + parts = rest.split('/', 1) + if len(parts) == 1: + raise click.BadArgumentUsage( + "Registry specification should be docker:REGISTRY/PATH[:TAG]") + + registry, path = parts + parts = path.split(':', 1) + if len(parts) == 1: + repo, tag = parts[0], 'latest' + else: + repo, tag = parts + + return RegistrySpec(registry, repo, tag, creds, tls_verify, cert_dir) + else: + raise click.BadArgumentUsage("Unknown source/destination: {}".format(spec)) + + +class RegistrySession(object): + def __init__(self, registry, insecure=False, creds=None, cert_dir=None): + self.registry = registry + self._resolved = None + self.insecure = insecure + + self.cert = self._find_cert(cert_dir) + + self.auth = None + if creds is not None: + username, password = creds.split(':', 1) + self.auth = requests.auth.HTTPBasicAuth(username, password) + + self._fallback = None + self._base = 'https://{}'.format(self.registry) + if insecure: + # In the insecure case, if the registry is just a hostname:port, we + # don't know whether to talk HTTPS or HTTP to it, so we try first + # with https then fallback + self._fallback = 'http://{}'.format(self.registry) + + self.session = requests.Session() + + def _find_cert_dir(self): + hostport = self.registry + + for d in ('/etc/containers/certs.d', '/etc/docker/certs.d'): + certs_dir = os.path.join(d, hostport) + if os.path.isdir(certs_dir): + return certs_dir + + return None + + def _find_cert(self, cert_dir): + if cert_dir is None: + cert_dir = self._find_cert_dir() + + if cert_dir is None: + return None + + for l in sorted(os.listdir(cert_dir)): + if l.endswith('.cert'): + certpath = os.path.join(cert_dir, l) + keypath = certpath[:-5] + '.key' + if not os.path.exists(keypath): + raise RuntimeError("Cannot find key file for {}".format(certpath)) + return (certpath, keypath) + elif l.endswith('.key'): + # Should have found .cert first + keypath = os.path.join(cert_dir, l) + raise RuntimeError("Cannot find certificate file for {}".format(keypath)) + + return None + + def _do(self, f, relative_url, *args, **kwargs): + kwargs['auth'] = self.auth + kwargs['cert'] = self.cert + kwargs['verify'] = not self.insecure + res = None + if self._fallback: + try: + res = f(self._base + relative_url, *args, **kwargs) + self._fallback = None # don't fallback after one success + except (SSLError, ConnectionError): + self._base = self._fallback + self._fallback = None + if res is None: + res = f(self._base + relative_url, *args, **kwargs) + return res + + def get(self, relative_url, data=None, **kwargs): + return self._do(self.session.get, relative_url, **kwargs) + + def head(self, relative_url, data=None, **kwargs): + return self._do(self.session.head, relative_url, **kwargs) + + def post(self, relative_url, data=None, **kwargs): + return self._do(self.session.post, relative_url, data=data, **kwargs) + + def put(self, relative_url, data=None, **kwargs): + return self._do(self.session.put, relative_url, data=data, **kwargs) + + +class ManifestInfo(object): + def __init__(self, contents, digest, media_type, size): + self.contents = contents + self.digest = digest + self.media_type = media_type + self.size = size + + +def get_manifest(session, repository, ref): + """Download a manifest from a registry. ref can be a digest, or a tag.""" + logger.debug("%s: Retrieving manifest for %s:%s", session.registry, repository, ref) + + headers = { + 'Accept': ', '.join(( + MEDIA_TYPE_MANIFEST_V2, + MEDIA_TYPE_LIST_V2, + MEDIA_TYPE_OCI, + MEDIA_TYPE_OCI_INDEX + )) + } + + url = '/v2/{}/manifests/{}'.format(repository, ref) + response = session.get(url, headers=headers) + response.raise_for_status() + return ManifestInfo(response.content, + response.headers['Docker-Content-Digest'], + response.headers['Content-Type'], + int(response.headers['Content-Length'])) + + +class DirectoryEndpoint(object): + def __init__(self, directory): + self.directory = directory + + def start(self): + with open(os.path.join(self.directory, 'oci-layout'), 'w') as f: + f.write('{"imageLayoutVersion": "1.0.0"}\n') + + def get_blob_path(self, digest): + algorithm, digest = digest.split(':', 2) + return os.path.join(self.directory, 'blobs', algorithm, digest) + + def ensure_blob_path(self, digest): + path = self.get_blob_path(digest) + + parent = os.path.dirname(path) + if not os.path.exists(parent): + os.makedirs(parent) + + return path + + def get_blob(self, digest): + with open(self.get_blob_path(digest), 'rb') as f: + return f.read() + + def has_blob(self, digest): + return os.path.exists(self.get_blob_path(digest)) + + def write_blob(self, digest, contents): + path = self.ensure_blob_path(digest) + + with open(path, 'wb') as f: + f.write(contents) + + def get_manifest(self, digest=None, media_type=None): + if digest is not None: + contents = self.get_blob(digest) + else: + manifest_path = os.path.join(self.directory, 'manifest.json') + if os.path.exists(manifest_path): + with open(manifest_path, 'rb') as f: + contents = f.read() + parsed = json.loads(contents) + media_type = parsed.get('mediaType', MEDIA_TYPE_OCI) + else: + index_path = os.path.join(self.directory, 'index.json') + with open(index_path, 'rb') as f: + contents = f.read() + parsed = json.loads(contents) + media_type = parsed.get('mediaType', MEDIA_TYPE_OCI_INDEX) + + return ManifestInfo(contents, digest, media_type, len(contents)) + + def write_manifest(self, info, toplevel=False): + if not toplevel: + self.write_blob(info.digest, info.contents) + elif info.media_type in (MEDIA_TYPE_LIST_V2, MEDIA_TYPE_OCI_INDEX): + with open(os.path.join(self.directory, 'index.json'), 'wb') as f: + f.write(info.contents) + else: + with open(os.path.join(self.directory, 'manifest.json'), 'wb') as f: + f.write(info.contents) + + +class RegistryEndpoint(object): + def __init__(self, spec): + self.session = spec.get_session() + self.registry = spec.registry + self.repo = spec.repo + self.tag = spec.tag + + def start(self): + pass + + def download_blob(self, digest, size, blob_path): + logger.info("%s: Downloading %s (size=%s)", self.registry, blob_path, size) + + url = "/v2/{}/blobs/{}".format(self.repo, digest) + result = self.session.get(url, stream=True) + result.raise_for_status() + + try: + with open(blob_path, 'wb') as f: + for block in result.iter_content(10 * 1024): + f.write(block) + finally: + result.close() + + def upload_blob(self, digest, size, blob_path): + logger.info("%s: Uploading %s (size=%s)", self.registry, blob_path, size) + + url = "/v2/{}/blobs/uploads/".format(self.repo) + result = self.session.post(url, data='') + result.raise_for_status() + + if result.status_code != requests.codes.ACCEPTED: + raise RuntimeError("Unexpected successful response %s", result.status_code) + + upload_url = result.headers.get('Location') + parsed = urlparse(upload_url) + if parsed.query == '': + query = 'digest=' + digest + else: + query = parsed.query + '&digest=' + digest + relative = urlunparse(('', '', parsed.path, parsed.params, query, '')) + + headers = { + 'Content-Length': str(size), + 'Content-Type': 'application/octet-stream' + } + with open(blob_path, 'rb') as f: + result = self.session.put(relative, data=f, headers=headers) + + result.raise_for_status() + if result.status_code != requests.codes.CREATED: + raise RuntimeError("Unexpected successful response %s", result.status_code) + + def link_blob(self, digest, src_repo): + logger.info("%s: Linking blob %s from %s to %s", + self.registry, digest, src_repo, self.repo) + + # Check that it exists in the source repository + url = "/v2/{}/blobs/{}".format(src_repo, digest) + result = self.session.head(url) + result.raise_for_status() + + url = "/v2/{}/blobs/uploads/?mount={}&from={}".format(self.repo, digest, src_repo) + result = self.session.post(url, data='') + result.raise_for_status() + + if result.status_code != requests.codes.CREATED: + # A 202-Accepted would mean that the source blob didn't exist and + # we're starting an upload - but we've checked that above + raise RuntimeError("Blob mount had unexpected status {}".format(result.status_code)) + + def has_blob(self, digest): + url = "/v2/{}/blobs/{}".format(self.repo, digest) + result = self.session.head(url, stream=True) + if result.status_code == 404: + return False + result.raise_for_status() + return True + + def get_manifest(self, digest=None, media_type=None): + if digest is None: + return get_manifest(self.session, self.repo, self.tag) + else: + return get_manifest(self.session, self.repo, digest) + + def write_manifest(self, info, toplevel=False, arch=None): + if toplevel: + ref = self.tag + else: + ref = info.digest + + logger.info("%s: Storing manifest as %s", self.registry, ref) + + url = '/v2/{}/manifests/{}'.format(self.repo, ref) + headers = {'Content-Type': info.media_type} + response = self.session.put(url, data=info.contents, headers=headers) + response.raise_for_status() + + +class Copier(object): + def __init__(self, src, dest): + self.src = src + self.dest = dest + + def _copy_blob(self, digest, size): + if self.dest.has_blob(digest): + return + + if isinstance(self.src, RegistryEndpoint) and isinstance(self.dest, DirectoryEndpoint): + self.src.download_blob(digest, size, self.dest.ensure_blob_path(digest)) + elif isinstance(self.src, DirectoryEndpoint) and isinstance(self.dest, RegistryEndpoint): + self.dest.upload_blob(digest, size, self.src.get_blob_path(digest)) + elif isinstance(self.src, RegistryEndpoint) and isinstance(self.dest, RegistryEndpoint): + if self.src.registry == self.dest.registry: + self.dest.link_blob(digest, self.src.repo) + else: + raise AssertionError("Direct copying between repositories not implemented") + else: + raise AssertionError("Copying between directories not implemented") + + def _copy_manifest(self, info, toplevel=False): + references = [] + if info.media_type in (MEDIA_TYPE_MANIFEST_V2, MEDIA_TYPE_OCI): + manifest = json.loads(info.contents) + references.append((manifest['config']['digest'], manifest['config']['size'])) + for layer in manifest['layers']: + references.append((layer['digest'], layer['size'])) + else: + raise RuntimeError("Unhandled media type %s", info.media_type) + + for digest, size in references: + self._copy_blob(digest, size) + + self.dest.write_manifest(info, toplevel=toplevel) + + def copy(self): + self.dest.start() + info = self.src.get_manifest() + if info.media_type in (MEDIA_TYPE_MANIFEST_V2, MEDIA_TYPE_OCI): + self._copy_manifest(info, toplevel=True) + elif info.media_type in (MEDIA_TYPE_LIST_V2, MEDIA_TYPE_OCI_INDEX): + manifest = json.loads(info.contents) + for m in manifest['manifests']: + referenced = self.src.get_manifest(digest=m['digest'], media_type=m['mediaType']) + self._copy_manifest(referenced) + self.dest.write_manifest(info, toplevel=True) + + else: + raise RuntimeError("Unhandled media type %s", info.media_type) + + +@main.command() +@click.option('--src-creds', '--screds', metavar='USERNAME[:PASSWORD]', + help='Use USERNAME[:PASSWORD] for accessing the source registry') +@click.option('--src-tls-verify', type=bool, default=True, + help='require HTTPS and verify certificates when talking to the ' + + 'container source registry (defaults to true)') +@click.option('--src-cert-dir', metavar='PATH', + help='use certificates at PATH (*.crt, *.cert, *.key) to connect to the ' + + 'source registry') +@click.option('--dest-creds', '--dcreds', metavar='USERNAME[:PASSWORD]', + help='Use USERNAME[:PASSWORD] for accessing the destination registry') +@click.option('--dest-tls-verify', type=bool, default=True, + help='require HTTPS and verify certificates when talking to the ' + + 'container destination registry (defaults to true)') +@click.option('--dest-cert-dir', metavar='PATH', + help='use certificates at PATH (*.crt, *.cert, *.key) to connect to the ' + + 'destination registry') +@click.argument('src', metavar='SOURCE-IMAGE-NAME') +@click.argument('dest', metavar='DEST-IMAGE-NAME') +def copy(src, dest, screds, src_tls_verify, src_cert_dir, dcreds, dest_tls_verify, dest_cert_dir): + """Copy an image from one location to another.""" + src = parse_spec(src, screds, src_tls_verify, src_cert_dir) + dest = parse_spec(dest, dcreds, dest_tls_verify, dest_cert_dir) + + if src.registry != dest.registry: + tempdir = tempfile.mkdtemp() + try: + tmp = DirectoryEndpoint(tempdir) + Copier(src.get_endpoint(), tmp).copy() + Copier(tmp, dest.get_endpoint()).copy() + finally: + shutil.rmtree(tempdir) + else: + Copier(src.get_endpoint(), dest.get_endpoint()).copy() + + +if __name__ == '__main__': + main() + +__all__ = ['copy', 'main'] diff --git a/bodhi/tests/server/scripts/test_skopeo_lite.py b/bodhi/tests/server/scripts/test_skopeo_lite.py new file mode 100644 index 0000000000..401004173d --- /dev/null +++ b/bodhi/tests/server/scripts/test_skopeo_lite.py @@ -0,0 +1,727 @@ +# -*- coding: utf-8 -*- +# Copyright © 2018 Red Hat, Inc. +# +# This file is part of Bodhi. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +"""This module contains tests for bodhi.server.scripts.skopeo_lite.""" + +from click import testing +from contextlib import contextmanager +from base64 import b64encode +import hashlib +import json +import mock +import os +import pytest +import re +import responses +import requests +from six import binary_type, text_type +from six.moves.urllib.parse import urlparse +import shutil +import tempfile +import uuid + +from bodhi.server.scripts import skopeo_lite +from bodhi.server.scripts.skopeo_lite import (MEDIA_TYPE_MANIFEST_V2, MEDIA_TYPE_LIST_V2, + MEDIA_TYPE_OCI, MEDIA_TYPE_OCI_INDEX) + + +REGISTRY_V1 = 'registry_v1.example.com' +REGISTRY_V2 = 'registry_v2.example.com' +OTHER_V2 = 'registry.example.com:5001' + +all_registry_conf = { + REGISTRY_V2: {'version': 'v2', 'insecure': True}, + OTHER_V2: {'version': 'v2', 'insecure': False}, +} + + +def registry_hostname(registry): + """ + Strip a reference to a registry to just the hostname:port + """ + if registry.startswith('http:') or registry.startswith('https:'): + return urlparse(registry).netloc + else: + return registry + + +def to_bytes(value): + if isinstance(value, binary_type): + return value + else: + return value.encode('utf-8') + + +def to_text(value): + if isinstance(value, text_type): + return value + else: + return text_type(value, 'utf-8') + + +def make_digest(blob): + # Abbreviate the hexdigest for readability of debugging output if things fail + return 'sha256:' + hashlib.sha256(to_bytes(blob)).hexdigest()[0:10] + + +class MockRegistry(object): + """ + This class mocks a subset of the v2 Docker Registry protocol. It also has methods to inject + and test content in the registry. + """ + def __init__(self, registry, insecure=False, required_creds=None, flags=''): + self.hostname = registry_hostname(registry) + self.insecure = insecure + self.repos = {} + self.required_creds = required_creds + self.flags = flags + self._add_pattern(responses.GET, r'/v2/(.*)/manifests/([^/]+)', + self._get_manifest) + self._add_pattern(responses.HEAD, r'/v2/(.*)/manifests/([^/]+)', + self._get_manifest) + self._add_pattern(responses.PUT, r'/v2/(.*)/manifests/([^/]+)', + self._put_manifest) + self._add_pattern(responses.GET, r'/v2/(.*)/blobs/([^/]+)', + self._get_blob) + self._add_pattern(responses.HEAD, r'/v2/(.*)/blobs/([^/]+)', + self._get_blob) + self._add_pattern(responses.POST, r'/v2/(.*)/blobs/uploads/', + self._post_blob) + self._add_pattern(responses.PUT, r'/v2/(.*)/blobs/uploads/([^?]*)\?digest=(.*)', + self._put_blob) + self._add_pattern(responses.PUT, r'/v2/(.*)/blobs/uploads/([^?]*)\?dummy=1&digest=(.*)', + self._put_blob) + self._add_pattern(responses.POST, r'/v2/(.*)/blobs/uploads/\?mount=([^&]+)&from=(.+)', + self._mount_blob) + + def get_repo(self, name): + return self.repos.setdefault(name, { + 'blobs': {}, + 'manifests': {}, + 'tags': {}, + 'uploads': {}, + }) + + def add_blob(self, name, blob): + repo = self.get_repo(name) + digest = make_digest(blob) + repo['blobs'][digest] = blob + return digest + + def get_blob(self, name, digest): + return self.get_repo(name)['blobs'][digest] + + def add_manifest(self, name, ref, manifest): + repo = self.get_repo(name) + digest = make_digest(manifest) + repo['manifests'][digest] = manifest + if ref is None: + pass + elif ref.startswith('sha256:'): + assert ref == digest + else: + repo['tags'][ref] = digest + return digest + + def get_manifest(self, name, ref): + repo = self.get_repo(name) + if not ref.startswith('sha256:'): + ref = repo['tags'][ref] + return repo['manifests'][ref] + + def _check_creds(self, req): + if self.required_creds: + username, password = self.required_creds + + auth = req.headers['Authorization'].strip().split() + assert auth[0] == 'Basic' + assert to_bytes(auth[1]) == b64encode(to_bytes(username + ':' + password)) + + def _add_pattern(self, method, pattern, callback): + if self.insecure: + url = 'http://' + self.hostname + else: + url = 'https://' + self.hostname + pat = re.compile('^' + url + pattern + '$') + + def do_it(req): + self._check_creds(req) + + status, headers, body = callback(req, *(pat.match(req.url).groups())) + if method == responses.HEAD: + return status, headers, '' + else: + return status, headers, body + + responses.add_callback(method, pat, do_it, match_querystring=True) + + def _get_manifest(self, req, name, ref): + repo = self.get_repo(name) + if not ref.startswith('sha256:'): + try: + ref = repo['tags'][ref] + except KeyError: + return (requests.codes.NOT_FOUND, {}, {'error': 'NOT_FOUND'}) + + try: + blob = repo['manifests'][ref] + except KeyError: + return (requests.codes.NOT_FOUND, {}, {'error': 'NOT_FOUND'}) + + decoded = json.loads(to_text(blob)) + content_type = decoded.get('mediaType') + if content_type is None: # OCI + if decoded.get('manifests') is not None: + content_type = MEDIA_TYPE_OCI_INDEX + else: + content_type = MEDIA_TYPE_OCI + + accepts = re.split('\s*,\s*', req.headers['Accept']) + assert content_type in accepts + + if 'bad_index_content_type' in self.flags: + if content_type == MEDIA_TYPE_OCI_INDEX: + content_type = 'application/json' + if 'bad_content_type' in self.flags: + if content_type == MEDIA_TYPE_OCI: + content_type = 'application/json' + + headers = { + 'Docker-Content-Digest': ref, + 'Content-Type': content_type, + 'Content-Length': str(len(blob)), + } + return (200, headers, blob) + + def _put_manifest(self, req, name, ref): + try: + json.loads(to_text(req.body)) + except ValueError: + return (400, {}, {'error': 'BAD_MANIFEST'}) + + self.add_manifest(name, ref, req.body) + return (200, {}, '') + + def _get_blob(self, req, name, digest): + repo = self.get_repo(name) + assert digest.startswith('sha256:') + + try: + blob = repo['blobs'][digest] + except KeyError: + return (requests.codes.NOT_FOUND, {}, {'error': 'NOT_FOUND'}) + + headers = { + 'Docker-Content-Digest': digest, + 'Content-Type': 'application/json', + 'Content-Length': str(len(blob)), + } + return (200, headers, blob) + + def _post_blob(self, req, name): + repo = self.get_repo(name) + uuid_str = str(uuid.uuid4()) + repo['uploads'][uuid_str] = '' + + if 'include_query_parameters' in self.flags: + location = '/v2/{}/blobs/uploads/{}?dummy=1'.format(name, uuid_str) + else: + location = '/v2/{}/blobs/uploads/{}'.format(name, uuid_str) + + headers = { + 'Location': location, + 'Range': 'bytes=0-0', + 'Content-Length': '0', + 'Docker-Upload-UUID': uuid_str, + } + return (200 if 'bad_post_status' in self.flags else 202, headers, '') + + def _put_blob(self, req, name, uuid, digest): + repo = self.get_repo(name) + + assert uuid in repo['uploads'] + del repo['uploads'][uuid] + + if isinstance(req.body, binary_type) or isinstance(req.body, text_type): + blob = req.body + else: + blob = req.body.read() + + added_digest = self.add_blob(name, blob) + assert added_digest == digest + + headers = { + 'Location': '/v2/{}/blobs/{}'.format(name, digest), + 'Docker-Content-Digest': added_digest, + } + + return (200 if 'bad_put_status' in self.flags else 201, headers, '') + + def _mount_blob(self, req, target_name, digest, source_name): + source_repo = self.get_repo(source_name) + target_repo = self.get_repo(target_name) + + try: + target_repo['blobs'][digest] = source_repo['blobs'][digest] + headers = { + 'Location': '/v2/{}/blobs/{}'.format(target_name, digest), + 'Docker-Content-Digest': digest, + } + return (200 if 'bad_mount_status' in self.flags else 201, headers, '') + except KeyError: + headers = { + 'Location': '/v2/{}/blobs/uploads/some-uuid'.format(target_name), + 'Docker-Upload-UUID': 'some-uuid', + } + return (202, headers, '') + + def add_fake_image(self, name, tag, content_type, + arch='amd64'): + layer_digest = self.add_blob(name, 'layer-' + arch) + layer_size = len(to_bytes('layer-' + arch)) + + config = { + 'architecture': arch, + 'os': 'linux', + } + config_bytes = to_bytes(json.dumps(config)) + config_digest = self.add_blob(name, config_bytes) + config_size = len(config_bytes) + + if content_type in (MEDIA_TYPE_MANIFEST_V2, MEDIA_TYPE_LIST_V2): + manifest = { + 'schemaVersion': 2, + 'mediaType': MEDIA_TYPE_MANIFEST_V2, + 'config': { + 'mediaType': 'application/vnd.docker.container.image.v1+json', + 'digest': config_digest, + 'size': config_size, + }, + 'layers': [{ + 'mediaType': 'application/vnd.docker.image.rootfs.diff.tar.gzip', + 'digest': layer_digest, + 'size': layer_size, + }] + } + + if content_type == MEDIA_TYPE_LIST_V2: + manifest_bytes = to_bytes(json.dumps(manifest)) + manifest_digest = self.add_manifest(name, None, manifest_bytes) + + manifest = { + 'schemaVersion': 2, + 'mediaType': MEDIA_TYPE_LIST_V2, + 'manifests': [{ + 'mediaType': MEDIA_TYPE_MANIFEST_V2, + 'size': len(manifest_bytes), + 'digest': manifest_digest, + 'platform': { + 'architecture': arch, + 'os': 'linux', + } + }] + } + else: + manifest = { + 'schemaVersion': 2, + 'mediaType': MEDIA_TYPE_OCI, + 'config': { + 'mediaType': 'application/vnd.oci.image.config.v1+json', + 'digest': config_digest, + 'size': config_size, + }, + 'layers': [{ + 'mediaType': 'application/vnd.oci.image.layer.v1.tar', + 'digest': layer_digest, + 'size': layer_size, + }] + } + + if content_type == MEDIA_TYPE_OCI_INDEX: + manifest_bytes = to_bytes(json.dumps(manifest)) + manifest_digest = self.add_manifest(name, None, manifest_bytes) + + manifest = { + 'schemaVersion': 2, + 'manifests': [{ + 'mediaType': MEDIA_TYPE_OCI, + 'size': len(manifest_bytes), + 'digest': manifest_digest, + 'platform': { + 'architecture': arch, + 'os': 'linux', + } + }] + } + + manifest_bytes = to_bytes(json.dumps(manifest)) + return self.add_manifest(name, tag, manifest_bytes) + + def check_fake_image(self, name, tag, digest, content_type): + manifest_bytes = self.get_manifest(name, tag) + assert make_digest(manifest_bytes) == digest + + manifest = json.loads(to_text(manifest_bytes)) + if content_type in (MEDIA_TYPE_LIST_V2, MEDIA_TYPE_OCI_INDEX): + manifest_digest = manifest['manifests'][0]['digest'] + manifest_bytes = self.get_manifest(name, manifest_digest) + manifest = json.loads(to_text(manifest_bytes)) + + config_digest = manifest['config']['digest'] + assert make_digest(self.get_blob(name, config_digest)) == config_digest + + layer_digest = manifest['layers'][0]['digest'] + assert make_digest(self.get_blob(name, layer_digest)) == layer_digest + + +@responses.activate +@pytest.mark.parametrize('content_type', + (MEDIA_TYPE_OCI, MEDIA_TYPE_OCI_INDEX, + MEDIA_TYPE_MANIFEST_V2, MEDIA_TYPE_LIST_V2)) +def test_skopeo_copy_basic(content_type): + """ + Test copying from one server to another + """ + runner = testing.CliRunner() + + reg1 = MockRegistry('registry1.example.com') + reg2 = MockRegistry('registry2.example.com') + digest = reg1.add_fake_image('repo1', 'latest', content_type) + + result = runner.invoke( + skopeo_lite.copy, + ['docker:registry1.example.com/repo1:latest', 'docker:registry2.example.com/repo2:latest'], + catch_exceptions=False) + + assert result.exit_code == 0 + + reg2.check_fake_image('repo2', 'latest', digest, content_type) + + +@responses.activate +@pytest.mark.parametrize('content_type', + (MEDIA_TYPE_OCI, MEDIA_TYPE_OCI_INDEX, + MEDIA_TYPE_MANIFEST_V2, MEDIA_TYPE_LIST_V2)) +def test_skopeo_copy_link(content_type): + """ + Testing copying on the same server, avoiding download/upload + """ + runner = testing.CliRunner() + + reg1 = MockRegistry('registry1.example.com') + digest = reg1.add_fake_image('repo1', 'latest', content_type) + + result = runner.invoke( + skopeo_lite.copy, + ['docker:registry1.example.com/repo1:latest', 'docker:registry1.example.com/repo2:latest'], + catch_exceptions=False) + + assert result.exit_code == 0 + + reg1.check_fake_image('repo2', 'latest', digest, content_type) + + +@responses.activate +@pytest.mark.parametrize('content_type', + (MEDIA_TYPE_OCI, MEDIA_TYPE_OCI_INDEX, + MEDIA_TYPE_MANIFEST_V2, MEDIA_TYPE_LIST_V2)) +def test_skopeo_copy_tag(content_type): + """ + Testing copying within the same repo - creating a new tag for an existing image + """ + runner = testing.CliRunner() + + reg1 = MockRegistry('registry1.example.com') + digest = reg1.add_fake_image('repo1', '1.2.3', content_type) + + # No tag should be the same as :latest + result = runner.invoke( + skopeo_lite.copy, + ['docker:registry1.example.com/repo1:1.2.3', 'docker:registry1.example.com/repo1'], + catch_exceptions=False) + + assert result.exit_code == 0 + + reg1.check_fake_image('repo1', 'latest', digest, content_type) + + +@responses.activate +@pytest.mark.parametrize('insecure', (True, False)) +def test_skopeo_copy_insecure(insecure): + """ + Testing falling back to HTTP when talking to a server + """ + runner = testing.CliRunner() + + content_type = MEDIA_TYPE_MANIFEST_V2 + reg1 = MockRegistry('registry1.example.com', insecure=insecure) + digest = reg1.add_fake_image('repo1', '1.2.3', content_type) + + result = runner.invoke( + skopeo_lite.copy, + ['--src-tls-verify', 'false', + '--dest-tls-verify', 'false', + 'docker:registry1.example.com/repo1:1.2.3', 'docker:registry1.example.com/repo1:latest'], + catch_exceptions=False) + + assert result.exit_code == 0 + + reg1.check_fake_image('repo1', 'latest', digest, content_type) + + +@responses.activate +def test_skopeo_copy_username_password(): + """ + Testing authentication with username and password + """ + runner = testing.CliRunner() + + content_type = MEDIA_TYPE_MANIFEST_V2 + reg1 = MockRegistry('registry1.example.com', required_creds=('someuser', 'somepassword')) + reg2 = MockRegistry('registry2.example.com', required_creds=('otheruser', 'otherpassword')) + digest = reg1.add_fake_image('repo1', 'latest', content_type) + + result = runner.invoke( + skopeo_lite.copy, + ['--src-creds', 'someuser:somepassword', + '--dest-creds', 'otheruser:otherpassword', + 'docker:registry1.example.com/repo1:latest', 'docker:registry2.example.com/repo2:latest'], + catch_exceptions=False) + + assert result.exit_code == 0 + + reg2.check_fake_image('repo2', 'latest', digest, content_type) + + +@contextmanager +def check_certificates(get_cert=None, put_cert=None): + old_get = requests.Session.get + old_put = requests.Session.put + + def checked_get(self, *args, **kwargs): + if kwargs.get('cert') != get_cert: + raise RuntimeError("Wrong/missing cert for GET") + + return old_get(self, *args, **kwargs) + + def checked_put(self, *args, **kwargs): + if kwargs.get('cert') != put_cert: + raise RuntimeError("Wrong/missing cert for PUT") + + return old_put(self, *args, **kwargs) + + with mock.patch('requests.Session.get', autospec=True, side_effect=checked_get): + with mock.patch('requests.Session.put', autospec=True, side_effect=checked_put): + yield + + +@responses.activate +@pytest.mark.parametrize(('breakage', 'error'), [ + (None, None), + ('missing_cert', 'Cannot find certificate file'), + ('missing_key', 'Cannot find key file'), + ('missing_cert_and_key', 'Wrong/missing cert'), +]) +def test_skopeo_copy_cert(breakage, error): + """ + Test authentication with a certificate + """ + runner = testing.CliRunner() + + certdir1 = tempfile.mkdtemp() + certdir2 = tempfile.mkdtemp() + try: + certs = {} + for certdir, reg in ((certdir1, 'registry1.example.com'), + (certdir2, 'registry2.example.com')): + cert = os.path.join(certdir, reg + '.cert') + if breakage not in ('missing_cert', 'missing_cert_and_key'): + with open(cert, 'w'): + pass + key = os.path.join(certdir, reg + '.key') + if breakage not in ('missing_key', 'missing_cert_and_key'): + with open(key, 'w'): + pass + certs[reg] = (cert, key) + + content_type = MEDIA_TYPE_MANIFEST_V2 + reg1 = MockRegistry('registry1.example.com') + reg2 = MockRegistry('registry2.example.com') + digest = reg1.add_fake_image('repo1', 'latest', content_type) + + with check_certificates(get_cert=certs['registry1.example.com'], + put_cert=certs['registry2.example.com']): + args = ['--src-cert-dir', certdir1, + '--dest-cert-dir', certdir2, + 'docker:registry1.example.com/repo1:latest', + 'docker:registry2.example.com/repo2:latest'] + + if breakage is None: + result = runner.invoke( + skopeo_lite.copy, + args, + catch_exceptions=False) + + assert result.exit_code == 0 + reg2.check_fake_image('repo2', 'latest', digest, content_type) + else: + with pytest.raises(Exception) as e: + runner.invoke( + skopeo_lite.copy, + args, + catch_exceptions=False) + assert error in str(e) + finally: + shutil.rmtree(certdir1) + shutil.rmtree(certdir2) + + +@contextmanager +def mock_system_certs(): + old_isdir = os.path.isdir + old_listdir = os.listdir + old_exists = os.path.exists + + def isdir(path): + if isinstance(path, text_type) and path.startswith('/etc/'): + return path in ('/etc/docker/certs.d/registry1.example.com', + '/etc/docker/certs.d/registry2.example.com') + else: + return old_isdir(path) + + def listdir(path): + if isinstance(path, text_type) and path.startswith('/etc/'): + if path in ('/etc/docker/certs.d/registry1.example.com', + '/etc/docker/certs.d/registry2.example.com'): + return ('client.cert', 'client.key') + else: + return None + else: + return old_listdir(path) + + def exists(path): + if isinstance(path, text_type) and path.startswith('/etc/'): + return path in ('/etc/docker/certs.d/registry1.example.com/client.cert', + '/etc/docker/certs.d/registry1.example.com/client.key', + '/etc/docker/certs.d/registry2.example.com/client.cert', + '/etc/docker/certs.d/registry2.example.com/client.key') + else: + return old_exists(path) + + with mock.patch('os.path.isdir', side_effect=isdir): + with mock.patch('os.listdir', side_effect=listdir): + with mock.patch('os.path.exists', side_effect=exists): + yield + + +@responses.activate +def test_skopeo_copy_system_cert(): + """ + Test using a certificate from a system directory + """ + runner = testing.CliRunner() + + reg1 = MockRegistry('registry1.example.com') + MockRegistry('registry2.example.com') + reg1.add_fake_image('repo1', 'latest', MEDIA_TYPE_MANIFEST_V2) + + with mock_system_certs(): + with check_certificates(get_cert=('/etc/docker/certs.d/registry1.example.com/client.cert', + '/etc/docker/certs.d/registry1.example.com/client.key'), + put_cert=('/etc/docker/certs.d/registry2.example.com/client.cert', + '/etc/docker/certs.d/registry2.example.com/client.key')): + runner.invoke( + skopeo_lite.copy, + ['docker:registry1.example.com/repo1:latest', + 'docker:registry2.example.com/repo2:latest'], + catch_exceptions=False) + + +@responses.activate +@pytest.mark.parametrize(('src', 'dest', 'error'), [ + ('docker:registry1.example.com/repo1:latest', 'badtype:registry2.example.com/repo2:latest', + 'Unknown source/destination'), + ('docker:registry1.example.com/repo1:latest', 'docker:registry2.example.com', + 'Registry specification should be docker:REGISTRY/PATH') +]) +def test_skopeo_copy_cli_errors(src, dest, error): + pass + """ + Test errors triggered from bad command line arguments + """ + runner = testing.CliRunner() + + options = [] + + result = runner.invoke( + skopeo_lite.copy, + options + [src, dest]) + + assert error in result.output + assert result.exit_code != 0 + + +@responses.activate +@pytest.mark.parametrize(('src', 'dest', 'flags1', 'flags2', 'error'), [ + ('docker:registry1.example.com/repo1:latest', 'docker:registry2.example.com/repo2:latest', + '', 'bad_put_status', + 'Unexpected successful response'), + ('docker:registry1.example.com/repo1:latest', 'docker:registry2.example.com/repo2:latest', + '', 'bad_post_status', + 'Unexpected successful response'), + ('docker:registry1.example.com/repo1:latest', 'docker:registry1.example.com/repo2:latest', + 'bad_mount_status', '', + 'Blob mount had unexpected status'), + ('docker:registry1.example.com/repo1:latest', 'docker:registry2.example.com/repo2:latest', + 'bad_index_content_type', '', + 'Unhandled media type'), + ('docker:registry1.example.com/repo1:latest', 'docker:registry2.example.com/repo2:latest', + 'bad_content_type', '', + 'Unhandled media type'), + ('docker:registry1.example.com/repo1:latest', 'docker:registry2.example.com/repo2:latest', + '', 'include_query_parameters', + None), +]) +def test_skopeo_copy_protocol(src, dest, flags1, flags2, error): + pass + """ + Tests various error and other code paths related to variations in server responses; the + flags argument to the MockRegistry constructor is used to modify server behavior. + """ + runner = testing.CliRunner() + + content_type = MEDIA_TYPE_OCI_INDEX + + reg1 = MockRegistry('registry1.example.com', flags=flags1) + MockRegistry('registry2.example.com', flags=flags2) + reg1.add_fake_image('repo1', 'latest', content_type) + + if error is not None: + with pytest.raises(Exception) as e: + runner.invoke( + skopeo_lite.copy, + [src, dest], + catch_exceptions=False) + + assert error in str(e) + else: + result = runner.invoke( + skopeo_lite.copy, + [src, dest]) + + assert result.exit_code == 0 diff --git a/devel/ci/pip-packages b/devel/ci/pip-packages index 1f7b78e293..47756e273b 100644 --- a/devel/ci/pip-packages +++ b/devel/ci/pip-packages @@ -15,6 +15,7 @@ RUN pip-2 install \ diff-cover \ flake8 \ mock \ + responses \ pytest \ pytest-cov \ sqlalchemy_schemadisplay \ @@ -22,6 +23,7 @@ RUN pip-2 install \ RUN pip-3 install \ diff-cover \ mock \ + responses \ pydocstyle \ pytest \ pytest-cov \ diff --git a/devel/ci/rpm-packages b/devel/ci/rpm-packages index f811906a89..7f742367a3 100644 --- a/devel/ci/rpm-packages +++ b/devel/ci/rpm-packages @@ -14,6 +14,7 @@ python2-pyramid-mako \ python2-pyramid-tm \ python2-pytest-cov \ + python2-responses \ python2-sqlalchemy \ python2-sqlalchemy_schemadisplay \ python3-alembic \ @@ -36,6 +37,7 @@ python3-pyramid-tm \ python3-pytest \ python3-pytest-cov \ + python3-responses \ python3-simplemediawiki \ python3-sqlalchemy \ python3-webtest \ diff --git a/production.ini b/production.ini index af62890f1d..8f75d64f5a 100644 --- a/production.ini +++ b/production.ini @@ -173,6 +173,13 @@ use = egg:bodhi-server # The skopeo executable to use to copy container images. # You can put credentials for skopeo to use in $HOME/.docker/config.json # https://github.com/projectatomic/skopeo#private-registries-with-authentication +# +# An alternative command is bodhi-skopeo-lite, installed by bodhi. It has various limitations +# (including not reading $HOME/.docker/config.json - it supports certificate authentication +# and credentials passed on the command line), but supports manifest lists and OCI image indexes, +# allowing copying multi-arch containers. See https://github.com/containers/image/pull/400 for +# work to add such support to skopeo proper. +# # skopeo.cmd = /usr/bin/skopeo # Comma separated list of extra flags to pass to the skopeo copy command. diff --git a/setup.py b/setup.py index f9788cf7cb..ab4801530c 100644 --- a/setup.py +++ b/setup.py @@ -162,6 +162,7 @@ def get_requirements(requirements_file='requirements.txt'): bodhi-approve-testing = bodhi.server.scripts.approve_testing:main bodhi-manage-releases = bodhi.server.scripts.manage_releases:main bodhi-check-policies = bodhi.server.scripts.check_policies:check + bodhi-skopeo-lite = bodhi.server.scripts.skopeo_lite:main [moksha.consumer] masher = bodhi.server.consumers.masher:Masher updates = bodhi.server.consumers.updates:UpdatesHandler