diff --git a/.coveragerc b/.coveragerc index 494941ba9d..5f7704bf3d 100644 --- a/.coveragerc +++ b/.coveragerc @@ -15,3 +15,4 @@ exclude_lines = def __repr__ def multicall_enabled if __name__ == .__main__.: + raise AssertionError \ No newline at end of file diff --git a/bodhi/server/scripts/skopeo_lite.py b/bodhi/server/scripts/skopeo_lite.py new file mode 100644 index 0000000000..852dffb9ae --- /dev/null +++ b/bodhi/server/scripts/skopeo_lite.py @@ -0,0 +1,471 @@ +# -*- coding: utf-8 -*- +# Copyright © 2018 Red Hat, Inc. and others. +# +# This file is part of Bodhi. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +""" +Copy containers between registries. This is a very limited version of the skopeo tool, +but with support for manifests lists and OCI image indexes. +https://github.com/containers/image/pull/400 will make this unnecessary. + +The only subcommand that is supported is 'copy', and the only supported image references +are Docker registries of the form 'docker://docker-reference'. + +No global options are supported, and only selected options to 'copy' are supported (see +--help for details.) + +Some other things that aren't implemented (but could be added if necessary): + - Handling of www-authenticate responses, necessary to log in to docker.io + - Special handling of 'docker.io' as 'registry-1.docker.io' + - Reading ~/.docker/config.json or $XDG_RUNTIME_DIR/containers/auth.json + - Handling foreign layers +""" + +import click +import json +import logging +import os +import requests +from requests.exceptions import SSLError, ConnectionError +import shutil +import tempfile +from six.moves.urllib.parse import urlparse, urlunparse + + +@click.group() +def main(): + """Simplified Skopeo work-alike with manifest list support.""" + pass # pragma: no cover + + +logger = logging.getLogger('skopeo-lite') +logging.basicConfig(level=logging.INFO) + + +MEDIA_TYPE_MANIFEST_V2 = 'application/vnd.docker.distribution.manifest.v2+json' +MEDIA_TYPE_LIST_V2 = 'application/vnd.docker.distribution.manifest.list.v2+json' +MEDIA_TYPE_OCI = 'application/vnd.oci.image.manifest.v1+json' +MEDIA_TYPE_OCI_INDEX = 'application/vnd.oci.image.index.v1+json' + + +class RegistrySpec(object): + def __init__(self, registry, repo, tag, creds, tls_verify, cert_dir): + self.registry = registry + self.repo = repo + self.tag = tag + self.creds = creds + self.cert_dir = cert_dir + self.tls_verify = tls_verify + + def get_session(self): + return RegistrySession(self.registry, insecure=not self.tls_verify, + creds=self.creds, cert_dir=self.cert_dir) + + def get_endpoint(self): + return RegistryEndpoint(self) + + +def parse_spec(spec, creds, tls_verify, cert_dir): + if spec.startswith('docker:'): + _, rest = spec.split(':', 1) + + parts = rest.split('/', 1) + if len(parts) == 1: + raise click.BadArgumentUsage( + "Registry specification should be docker:REGISTRY/PATH[:TAG]") + + registry, path = parts + parts = path.split(':', 1) + if len(parts) == 1: + repo, tag = parts[0], 'latest' + else: + repo, tag = parts + + return RegistrySpec(registry, repo, tag, creds, tls_verify, cert_dir) + else: + raise click.BadArgumentUsage("Unknown source/destination: {}".format(spec)) + + +class RegistrySession(object): + def __init__(self, registry, insecure=False, creds=None, cert_dir=None): + self.registry = registry + self._resolved = None + self.insecure = insecure + + self.cert = self._find_cert(cert_dir) + + self.auth = None + if creds is not None: + username, password = creds.split(':', 1) + self.auth = requests.auth.HTTPBasicAuth(username, password) + + self._fallback = None + self._base = 'https://{}'.format(self.registry) + if insecure: + # In the insecure case, if the registry is just a hostname:port, we + # don't know whether to talk HTTPS or HTTP to it, so we try first + # with https then fallback + self._fallback = 'http://{}'.format(self.registry) + + self.session = requests.Session() + + def _find_cert_dir(self): + hostport = self.registry + + for d in ('/etc/containers/certs.d', '/etc/docker/certs.d'): + certs_dir = os.path.join(d, hostport) + if os.path.isdir(certs_dir): + return certs_dir + + return None + + def _find_cert(self, cert_dir): + if cert_dir is None: + cert_dir = self._find_cert_dir() + + if cert_dir is None: + return None + + for l in sorted(os.listdir(cert_dir)): + if l.endswith('.cert'): + certpath = os.path.join(cert_dir, l) + keypath = certpath[:-5] + '.key' + if not os.path.exists(keypath): + raise RuntimeError("Cannot find key file for {}".format(certpath)) + return (certpath, keypath) + elif l.endswith('.key'): + # Should have found .cert first + keypath = os.path.join(cert_dir, l) + raise RuntimeError("Cannot find certificate file for {}".format(keypath)) + + return None + + def _do(self, f, relative_url, *args, **kwargs): + kwargs['auth'] = self.auth + kwargs['cert'] = self.cert + kwargs['verify'] = not self.insecure + res = None + if self._fallback: + try: + res = f(self._base + relative_url, *args, **kwargs) + self._fallback = None # don't fallback after one success + except (SSLError, ConnectionError): + self._base = self._fallback + self._fallback = None + if res is None: + res = f(self._base + relative_url, *args, **kwargs) + return res + + def get(self, relative_url, data=None, **kwargs): + return self._do(self.session.get, relative_url, **kwargs) + + def head(self, relative_url, data=None, **kwargs): + return self._do(self.session.head, relative_url, **kwargs) + + def post(self, relative_url, data=None, **kwargs): + return self._do(self.session.post, relative_url, data=data, **kwargs) + + def put(self, relative_url, data=None, **kwargs): + return self._do(self.session.put, relative_url, data=data, **kwargs) + + +class ManifestInfo(object): + def __init__(self, contents, digest, media_type, size): + self.contents = contents + self.digest = digest + self.media_type = media_type + self.size = size + + +def get_manifest(session, repository, ref): + """ + Downloads a manifest from a registry. ref can be a digest, or a tag. + """ + logger.debug("%s: Retrieving manifest for %s:%s", session.registry, repository, ref) + + headers = { + 'Accept': ', '.join(( + MEDIA_TYPE_MANIFEST_V2, + MEDIA_TYPE_LIST_V2, + MEDIA_TYPE_OCI, + MEDIA_TYPE_OCI_INDEX + )) + } + + url = '/v2/{}/manifests/{}'.format(repository, ref) + response = session.get(url, headers=headers) + response.raise_for_status() + return ManifestInfo(response.content, + response.headers['Docker-Content-Digest'], + response.headers['Content-Type'], + int(response.headers['Content-Length'])) + + +class DirectoryEndpoint(object): + def __init__(self, directory): + self.directory = directory + + def start(self): + with open(os.path.join(self.directory, 'oci-layout'), 'w') as f: + f.write('{"imageLayoutVersion": "1.0.0"}\n') + + def get_blob_path(self, digest): + algorithm, digest = digest.split(':', 2) + return os.path.join(self.directory, 'blobs', algorithm, digest) + + def ensure_blob_path(self, digest): + path = self.get_blob_path(digest) + + parent = os.path.dirname(path) + if not os.path.exists(parent): + os.makedirs(parent) + + return path + + def get_blob(self, digest): + with open(self.get_blob_path(digest), 'rb') as f: + return f.read() + + def has_blob(self, digest): + return os.path.exists(self.get_blob_path(digest)) + + def write_blob(self, digest, contents): + path = self.ensure_blob_path(digest) + + with open(path, 'wb') as f: + f.write(contents) + + def get_manifest(self, digest=None, media_type=None): + if digest is not None: + contents = self.get_blob(digest) + else: + manifest_path = os.path.join(self.directory, 'manifest.json') + if os.path.exists(manifest_path): + with open(manifest_path, 'rb') as f: + contents = f.read() + parsed = json.loads(contents) + media_type = parsed.get('mediaType', MEDIA_TYPE_OCI) + else: + index_path = os.path.join(self.directory, 'index.json') + with open(index_path, 'rb') as f: + contents = f.read() + parsed = json.loads(contents) + media_type = parsed.get('mediaType', MEDIA_TYPE_OCI_INDEX) + + return ManifestInfo(contents, digest, media_type, len(contents)) + + def write_manifest(self, info, toplevel=False): + if not toplevel: + self.write_blob(info.digest, info.contents) + elif info.media_type in (MEDIA_TYPE_LIST_V2, MEDIA_TYPE_OCI_INDEX): + with open(os.path.join(self.directory, 'index.json'), 'wb') as f: + f.write(info.contents) + else: + with open(os.path.join(self.directory, 'manifest.json'), 'wb') as f: + f.write(info.contents) + + +class RegistryEndpoint(object): + def __init__(self, spec): + self.session = spec.get_session() + self.registry = spec.registry + self.repo = spec.repo + self.tag = spec.tag + + def start(self): + pass + + def download_blob(self, digest, size, blob_path): + logger.info("%s: Downloading %s (size=%s)", self.registry, blob_path, size) + + url = "/v2/{}/blobs/{}".format(self.repo, digest) + result = self.session.get(url, stream=True) + result.raise_for_status() + + try: + with open(blob_path, 'wb') as f: + for block in result.iter_content(10 * 1024): + f.write(block) + finally: + result.close() + + def upload_blob(self, digest, size, blob_path): + logger.info("%s: Uploading %s (size=%s)", self.registry, blob_path, size) + + url = "/v2/{}/blobs/uploads/".format(self.repo) + result = self.session.post(url, data='') + result.raise_for_status() + + if result.status_code != requests.codes.ACCEPTED: + raise RuntimeError("Unexpected successful response %s", result.status_code) + + upload_url = result.headers.get('Location') + parsed = urlparse(upload_url) + if parsed.query == '': + query = 'digest=' + digest + else: + query = parsed.query + '&digest=' + digest + relative = urlunparse(('', '', parsed.path, parsed.params, query, '')) + + headers = { + 'Content-Length': str(size), + 'Content-Type': 'application/octet-stream' + } + with open(blob_path, 'rb') as f: + result = self.session.put(relative, data=f, headers=headers) + + result.raise_for_status() + if result.status_code != requests.codes.CREATED: + raise RuntimeError("Unexpected successful response %s", result.status_code) + + def link_blob(self, digest, src_repo): + logger.info("%s: Linking blob %s from %s to %s", + self.registry, digest, src_repo, self.repo) + + # Check that it exists in the source repository + url = "/v2/{}/blobs/{}".format(src_repo, digest) + result = self.session.head(url) + result.raise_for_status() + + url = "/v2/{}/blobs/uploads/?mount={}&from={}".format(self.repo, digest, src_repo) + result = self.session.post(url, data='') + result.raise_for_status() + + if result.status_code != requests.codes.CREATED: + # A 202-Accepted would mean that the source blob didn't exist and + # we're starting an upload - but we've checked that above + raise RuntimeError("Blob mount had unexpected status {}".format(result.status_code)) + + def has_blob(self, digest): + url = "/v2/{}/blobs/{}".format(self.repo, digest) + result = self.session.head(url, stream=True) + if result.status_code == 404: + return False + result.raise_for_status() + return True + + def get_manifest(self, digest=None, media_type=None): + if digest is None: + return get_manifest(self.session, self.repo, self.tag) + else: + return get_manifest(self.session, self.repo, digest) + + def write_manifest(self, info, toplevel=False, arch=None): + if toplevel: + ref = self.tag + else: + ref = info.digest + + logger.info("%s: Storing manifest as %s", self.registry, ref) + + url = '/v2/{}/manifests/{}'.format(self.repo, ref) + headers = {'Content-Type': info.media_type} + response = self.session.put(url, data=info.contents, headers=headers) + response.raise_for_status() + + +class Copier(object): + def __init__(self, src, dest): + self.src = src + self.dest = dest + + def _copy_blob(self, digest, size): + if self.dest.has_blob(digest): + return + + if isinstance(self.src, RegistryEndpoint) and isinstance(self.dest, DirectoryEndpoint): + self.src.download_blob(digest, size, self.dest.ensure_blob_path(digest)) + elif isinstance(self.src, DirectoryEndpoint) and isinstance(self.dest, RegistryEndpoint): + self.dest.upload_blob(digest, size, self.src.get_blob_path(digest)) + elif isinstance(self.src, RegistryEndpoint) and isinstance(self.dest, RegistryEndpoint): + if self.src.registry == self.dest.registry: + self.dest.link_blob(digest, self.src.repo) + else: + raise AssertionError("Direct copying between repositories not implemented") + else: + raise AssertionError("Copying between directories not implemented") + + def _copy_manifest(self, info, toplevel=False): + references = [] + if info.media_type in (MEDIA_TYPE_MANIFEST_V2, MEDIA_TYPE_OCI): + manifest = json.loads(info.contents) + references.append((manifest['config']['digest'], manifest['config']['size'])) + for layer in manifest['layers']: + references.append((layer['digest'], layer['size'])) + else: + raise RuntimeError("Unhandled media type %s", info.media_type) + + for digest, size in references: + self._copy_blob(digest, size) + + self.dest.write_manifest(info, toplevel=toplevel) + + def copy(self): + self.dest.start() + info = self.src.get_manifest() + if info.media_type in (MEDIA_TYPE_MANIFEST_V2, MEDIA_TYPE_OCI): + self._copy_manifest(info, toplevel=True) + elif info.media_type in (MEDIA_TYPE_LIST_V2, MEDIA_TYPE_OCI_INDEX): + manifest = json.loads(info.contents) + for m in manifest['manifests']: + referenced = self.src.get_manifest(digest=m['digest'], media_type=m['mediaType']) + self._copy_manifest(referenced) + self.dest.write_manifest(info, toplevel=True) + + else: + raise RuntimeError("Unhandled media type %s", info.media_type) + + +@main.command() +@click.option('--src-creds', '--screds', metavar='USERNAME[:PASSWORD]', + help='Use USERNAME[:PASSWORD] for accessing the source registry') +@click.option('--src-tls-verify', type=bool, default=True, + help='require HTTPS and verify certificates when talking to the ' + + 'container source registry (defaults to true)') +@click.option('--src-cert-dir', metavar='PATH', + help='use certificates at PATH (*.crt, *.cert, *.key) to connect to the ' + + 'source registry') +@click.option('--dest-creds', '--dcreds', metavar='USERNAME[:PASSWORD]', + help='Use USERNAME[:PASSWORD] for accessing the destination registry') +@click.option('--dest-tls-verify', type=bool, default=True, + help='require HTTPS and verify certificates when talking to the ' + + 'container destination registry (defaults to true)') +@click.option('--dest-cert-dir', metavar='PATH', + help='use certificates at PATH (*.crt, *.cert, *.key) to connect to the ' + + 'destination registry') +@click.argument('src', metavar='SOURCE-IMAGE-NAME') +@click.argument('dest', metavar='DEST-IMAGE-NAME') +def copy(src, dest, screds, src_tls_verify, src_cert_dir, dcreds, dest_tls_verify, dest_cert_dir): + """ + Copy an image from one location to another + """ + src = parse_spec(src, screds, src_tls_verify, src_cert_dir) + dest = parse_spec(dest, dcreds, dest_tls_verify, dest_cert_dir) + + if src.registry != dest.registry: + tempdir = tempfile.mkdtemp() + try: + tmp = DirectoryEndpoint(tempdir) + Copier(src.get_endpoint(), tmp).copy() + Copier(tmp, dest.get_endpoint()).copy() + finally: + shutil.rmtree(tempdir) + else: + Copier(src.get_endpoint(), dest.get_endpoint()).copy() + + +if __name__ == '__main__': + main() diff --git a/bodhi/tests/server/scripts/test_skopeo_lite.py b/bodhi/tests/server/scripts/test_skopeo_lite.py new file mode 100644 index 0000000000..335511ddae --- /dev/null +++ b/bodhi/tests/server/scripts/test_skopeo_lite.py @@ -0,0 +1,729 @@ +# -*- coding: utf-8 -*- +# Copyright © 2018 Red Hat, Inc. +# +# This file is part of Bodhi. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +"""This module contains tests for bodhi.server.scripts.skopeo_lite.""" + +from click import testing +from contextlib import contextmanager +from base64 import b64encode +import hashlib +import json +import mock +import os +import pytest +import re +import responses +import requests +from six import binary_type, text_type +from six.moves.urllib.parse import urlparse +import shutil +import sys +import tempfile +import uuid + +from bodhi.server.scripts import skopeo_lite +from bodhi.server.scripts.skopeo_lite import (MEDIA_TYPE_MANIFEST_V2, MEDIA_TYPE_LIST_V2, + MEDIA_TYPE_OCI, MEDIA_TYPE_OCI_INDEX) + + +REGISTRY_V1 = 'registry_v1.example.com' +REGISTRY_V2 = 'registry_v2.example.com' +OTHER_V2 = 'registry.example.com:5001' + +all_registry_conf = { + REGISTRY_V2: {'version': 'v2', 'insecure': True}, + OTHER_V2: {'version': 'v2', 'insecure': False}, +} + + +def registry_hostname(registry): + """ + Strip a reference to a registry to just the hostname:port + """ + if registry.startswith('http:') or registry.startswith('https:'): + return urlparse(registry).netloc + else: + return registry + + +def to_bytes(value): + if isinstance(value, binary_type): + return value + else: + return value.encode('utf-8') + + +def to_text(value): + if isinstance(value, text_type): + return value + else: + return text_type(value, 'utf-8') + + +def make_digest(blob): + # Abbreviate the hexdigest for readability of debugging output if things fail + return 'sha256:' + hashlib.sha256(to_bytes(blob)).hexdigest()[0:10] + + +class MockRegistry(object): + """ + This class mocks a subset of the v2 Docker Registry protocol. It also has methods to inject + and test content in the registry. + """ + def __init__(self, registry, insecure=False, required_creds=None, flags=''): + self.hostname = registry_hostname(registry) + self.insecure = insecure + self.repos = {} + self.required_creds = required_creds + self.flags = flags + self._add_pattern(responses.GET, r'/v2/(.*)/manifests/([^/]+)', + self._get_manifest) + self._add_pattern(responses.HEAD, r'/v2/(.*)/manifests/([^/]+)', + self._get_manifest) + self._add_pattern(responses.PUT, r'/v2/(.*)/manifests/([^/]+)', + self._put_manifest) + self._add_pattern(responses.GET, r'/v2/(.*)/blobs/([^/]+)', + self._get_blob) + self._add_pattern(responses.HEAD, r'/v2/(.*)/blobs/([^/]+)', + self._get_blob) + self._add_pattern(responses.POST, r'/v2/(.*)/blobs/uploads/', + self._post_blob) + self._add_pattern(responses.PUT, r'/v2/(.*)/blobs/uploads/([^?]*)\?digest=(.*)', + self._put_blob) + self._add_pattern(responses.PUT, r'/v2/(.*)/blobs/uploads/([^?]*)\?dummy=1&digest=(.*)', + self._put_blob) + self._add_pattern(responses.POST, r'/v2/(.*)/blobs/uploads/\?mount=([^&]+)&from=(.+)', + self._mount_blob) + + def get_repo(self, name): + return self.repos.setdefault(name, { + 'blobs': {}, + 'manifests': {}, + 'tags': {}, + 'uploads': {}, + }) + + def add_blob(self, name, blob): + repo = self.get_repo(name) + digest = make_digest(blob) + repo['blobs'][digest] = blob + return digest + + def get_blob(self, name, digest): + return self.get_repo(name)['blobs'][digest] + + def add_manifest(self, name, ref, manifest): + repo = self.get_repo(name) + digest = make_digest(manifest) + repo['manifests'][digest] = manifest + if ref is None: + pass + elif ref.startswith('sha256:'): + assert ref == digest + else: + repo['tags'][ref] = digest + return digest + + def get_manifest(self, name, ref): + repo = self.get_repo(name) + if not ref.startswith('sha256:'): + ref = repo['tags'][ref] + return repo['manifests'][ref] + + def _check_creds(self, req): + if self.required_creds: + username, password = self.required_creds + + auth = req.headers['Authorization'].strip().split() + assert auth[0] == 'Basic' + assert auth[1] == b64encode(username + ':' + password) + + def _add_pattern(self, method, pattern, callback): + if self.insecure: + url = 'http://' + self.hostname + else: + url = 'https://' + self.hostname + pat = re.compile('^' + url + pattern + '$') + + def do_it(req): + self._check_creds(req) + + status, headers, body = callback(req, *(pat.match(req.url).groups())) + if method == responses.HEAD: + return status, headers, '' + else: + return status, headers, body + + responses.add_callback(method, pat, do_it, match_querystring=True) + + def _get_manifest(self, req, name, ref): + repo = self.get_repo(name) + if not ref.startswith('sha256:'): + try: + ref = repo['tags'][ref] + except KeyError: + return (requests.codes.NOT_FOUND, {}, {'error': 'NOT_FOUND'}) + + try: + blob = repo['manifests'][ref] + except KeyError: + return (requests.codes.NOT_FOUND, {}, {'error': 'NOT_FOUND'}) + + decoded = json.loads(to_text(blob)) + content_type = decoded.get('mediaType') + if content_type is None: # OCI + if decoded.get('manifests') is not None: + content_type = MEDIA_TYPE_OCI_INDEX + else: + content_type = MEDIA_TYPE_OCI + + accepts = re.split('\s*,\s*', req.headers['Accept']) + assert content_type in accepts + + if 'bad_index_content_type' in self.flags: + if content_type == MEDIA_TYPE_OCI_INDEX: + content_type = 'application/json' + if 'bad_content_type' in self.flags: + if content_type == MEDIA_TYPE_OCI: + content_type = 'application/json' + + headers = { + 'Docker-Content-Digest': ref, + 'Content-Type': content_type, + 'Content-Length': str(len(blob)), + } + return (200, headers, blob) + + def _put_manifest(self, req, name, ref): + try: + json.loads(to_text(req.body)) + except ValueError: + return (400, {}, {'error': 'BAD_MANIFEST'}) + + self.add_manifest(name, ref, req.body) + return (200, {}, '') + + def _get_blob(self, req, name, digest): + repo = self.get_repo(name) + assert digest.startswith('sha256:') + + try: + blob = repo['blobs'][digest] + except KeyError: + return (requests.codes.NOT_FOUND, {}, {'error': 'NOT_FOUND'}) + + headers = { + 'Docker-Content-Digest': digest, + 'Content-Type': 'application/json', + 'Content-Length': str(len(blob)), + } + return (200, headers, blob) + + def _post_blob(self, req, name): + repo = self.get_repo(name) + uuid_str = str(uuid.uuid4()) + repo['uploads'][uuid_str] = '' + + if 'include_query_parameters' in self.flags: + location = '/v2/{}/blobs/uploads/{}?dummy=1'.format(name, uuid_str) + else: + location = '/v2/{}/blobs/uploads/{}'.format(name, uuid_str) + + headers = { + 'Location': location, + 'Range': 'bytes=0-0', + 'Content-Length': '0', + 'Docker-Upload-UUID': uuid_str, + } + return (200 if 'bad_post_status' in self.flags else 202, headers, '') + + def _put_blob(self, req, name, uuid, digest): + repo = self.get_repo(name) + + assert uuid in repo['uploads'] + del repo['uploads'][uuid] + + if isinstance(req.body, binary_type) or isinstance(req.body, text_type): + blob = req.body + else: + blob = req.body.read() + + added_digest = self.add_blob(name, blob) + assert added_digest == digest + + headers = { + 'Location': '/v2/{}/blobs/{}'.format(name, digest), + 'Docker-Content-Digest': added_digest, + } + + return (200 if 'bad_put_status' in self.flags else 201, headers, '') + + def _mount_blob(self, req, target_name, digest, source_name): + source_repo = self.get_repo(source_name) + target_repo = self.get_repo(target_name) + + try: + target_repo['blobs'][digest] = source_repo['blobs'][digest] + headers = { + 'Location': '/v2/{}/blobs/{}'.format(target_name, digest), + 'Docker-Content-Digest': digest, + } + return (200 if 'bad_mount_status' in self.flags else 201, headers, '') + except KeyError: + headers = { + 'Location': '/v2/{}/blobs/uploads/some-uuid'.format(target_name), + 'Docker-Upload-UUID': 'some-uuid', + } + return (202, headers, '') + + def add_fake_image(self, name, tag, content_type, + arch='amd64'): + layer_digest = self.add_blob(name, 'layer-' + arch) + layer_size = len(to_bytes('layer-' + arch)) + + config = { + 'architecture': arch, + 'os': 'linux', + } + config_bytes = to_bytes(json.dumps(config)) + config_digest = self.add_blob(name, config_bytes) + config_size = len(config_bytes) + + if content_type in (MEDIA_TYPE_MANIFEST_V2, MEDIA_TYPE_LIST_V2): + manifest = { + 'schemaVersion': 2, + 'mediaType': MEDIA_TYPE_MANIFEST_V2, + 'config': { + 'mediaType': 'application/vnd.docker.container.image.v1+json', + 'digest': config_digest, + 'size': config_size, + }, + 'layers': [{ + 'mediaType': 'application/vnd.docker.image.rootfs.diff.tar.gzip', + 'digest': layer_digest, + 'size': layer_size, + }] + } + + if content_type == MEDIA_TYPE_LIST_V2: + manifest_bytes = to_bytes(json.dumps(manifest)) + manifest_digest = self.add_manifest(name, None, manifest_bytes) + print >>sys.stderr, manifest_digest + + manifest = { + 'schemaVersion': 2, + 'mediaType': MEDIA_TYPE_LIST_V2, + 'manifests': [{ + 'mediaType': MEDIA_TYPE_MANIFEST_V2, + 'size': len(manifest_bytes), + 'digest': manifest_digest, + 'platform': { + 'architecture': arch, + 'os': 'linux', + } + }] + } + else: + manifest = { + 'schemaVersion': 2, + 'mediaType': MEDIA_TYPE_OCI, + 'config': { + 'mediaType': 'application/vnd.oci.image.config.v1+json', + 'digest': config_digest, + 'size': config_size, + }, + 'layers': [{ + 'mediaType': 'application/vnd.oci.image.layer.v1.tar', + 'digest': layer_digest, + 'size': layer_size, + }] + } + + if content_type == MEDIA_TYPE_OCI_INDEX: + manifest_bytes = to_bytes(json.dumps(manifest)) + manifest_digest = self.add_manifest(name, None, manifest_bytes) + + manifest = { + 'schemaVersion': 2, + 'manifests': [{ + 'mediaType': MEDIA_TYPE_OCI, + 'size': len(manifest_bytes), + 'digest': manifest_digest, + 'platform': { + 'architecture': arch, + 'os': 'linux', + } + }] + } + + manifest_bytes = to_bytes(json.dumps(manifest)) + return self.add_manifest(name, tag, manifest_bytes) + + def check_fake_image(self, name, tag, digest, content_type): + manifest_bytes = self.get_manifest(name, tag) + assert make_digest(manifest_bytes) == digest + + manifest = json.loads(to_text(manifest_bytes)) + if content_type in (MEDIA_TYPE_LIST_V2, MEDIA_TYPE_OCI_INDEX): + manifest_digest = manifest['manifests'][0]['digest'] + manifest_bytes = self.get_manifest(name, manifest_digest) + manifest = json.loads(to_text(manifest_bytes)) + + config_digest = manifest['config']['digest'] + assert make_digest(self.get_blob(name, config_digest)) == config_digest + + layer_digest = manifest['layers'][0]['digest'] + assert make_digest(self.get_blob(name, layer_digest)) == layer_digest + + +@responses.activate +@pytest.mark.parametrize('content_type', + (MEDIA_TYPE_OCI, MEDIA_TYPE_OCI_INDEX, + MEDIA_TYPE_MANIFEST_V2, MEDIA_TYPE_LIST_V2)) +def test_skopeo_copy_basic(content_type): + """ + Test copying from one server to another + """ + runner = testing.CliRunner() + + reg1 = MockRegistry('registry1.example.com') + reg2 = MockRegistry('registry2.example.com') + digest = reg1.add_fake_image('repo1', 'latest', content_type) + + result = runner.invoke( + skopeo_lite.copy, + ['docker:registry1.example.com/repo1:latest', 'docker:registry2.example.com/repo2:latest'], + catch_exceptions=False) + + assert result.exit_code == 0 + + reg2.check_fake_image('repo2', 'latest', digest, content_type) + + +@responses.activate +@pytest.mark.parametrize('content_type', + (MEDIA_TYPE_OCI, MEDIA_TYPE_OCI_INDEX, + MEDIA_TYPE_MANIFEST_V2, MEDIA_TYPE_LIST_V2)) +def test_skopeo_copy_link(content_type): + """ + Testing copying on the same server, avoiding download/upload + """ + runner = testing.CliRunner() + + reg1 = MockRegistry('registry1.example.com') + digest = reg1.add_fake_image('repo1', 'latest', content_type) + + result = runner.invoke( + skopeo_lite.copy, + ['docker:registry1.example.com/repo1:latest', 'docker:registry1.example.com/repo2:latest'], + catch_exceptions=False) + + assert result.exit_code == 0 + + reg1.check_fake_image('repo2', 'latest', digest, content_type) + + +@responses.activate +@pytest.mark.parametrize('content_type', + (MEDIA_TYPE_OCI, MEDIA_TYPE_OCI_INDEX, + MEDIA_TYPE_MANIFEST_V2, MEDIA_TYPE_LIST_V2)) +def test_skopeo_copy_tag(content_type): + """ + Testing copying within the same repo - creating a new tag for an existing image + """ + runner = testing.CliRunner() + + reg1 = MockRegistry('registry1.example.com') + digest = reg1.add_fake_image('repo1', '1.2.3', content_type) + + # No tag should be the same as :latest + result = runner.invoke( + skopeo_lite.copy, + ['docker:registry1.example.com/repo1:1.2.3', 'docker:registry1.example.com/repo1'], + catch_exceptions=False) + + assert result.exit_code == 0 + + reg1.check_fake_image('repo1', 'latest', digest, content_type) + + +@responses.activate +@pytest.mark.parametrize('insecure', (True, False)) +def test_skopeo_copy_insecure(insecure): + """ + Testing falling back to HTTP when talking to a server + """ + runner = testing.CliRunner() + + content_type = MEDIA_TYPE_MANIFEST_V2 + reg1 = MockRegistry('registry1.example.com', insecure=insecure) + digest = reg1.add_fake_image('repo1', '1.2.3', content_type) + + result = runner.invoke( + skopeo_lite.copy, + ['--src-tls-verify', 'false', + '--dest-tls-verify', 'false', + 'docker:registry1.example.com/repo1:1.2.3', 'docker:registry1.example.com/repo1:latest'], + catch_exceptions=False) + + assert result.exit_code == 0 + + reg1.check_fake_image('repo1', 'latest', digest, content_type) + + +@responses.activate +def test_skopeo_copy_username_password(): + """ + Testing authentication with username and password + """ + runner = testing.CliRunner() + + content_type = MEDIA_TYPE_MANIFEST_V2 + reg1 = MockRegistry('registry1.example.com', required_creds=('someuser', 'somepassword')) + reg2 = MockRegistry('registry2.example.com', required_creds=('otheruser', 'otherpassword')) + digest = reg1.add_fake_image('repo1', 'latest', content_type) + + result = runner.invoke( + skopeo_lite.copy, + ['--src-creds', 'someuser:somepassword', + '--dest-creds', 'otheruser:otherpassword', + 'docker:registry1.example.com/repo1:latest', 'docker:registry2.example.com/repo2:latest'], + catch_exceptions=False) + + assert result.exit_code == 0 + + reg2.check_fake_image('repo2', 'latest', digest, content_type) + + +@contextmanager +def check_certificates(get_cert=None, put_cert=None): + old_get = requests.Session.get + old_put = requests.Session.put + + def checked_get(self, *args, **kwargs): + if kwargs.get('cert') != get_cert: + raise RuntimeError("Wrong/missing cert for GET") + + return old_get(self, *args, **kwargs) + + def checked_put(self, *args, **kwargs): + if kwargs.get('cert') != put_cert: + raise RuntimeError("Wrong/missing cert for PUT") + + return old_put(self, *args, **kwargs) + + with mock.patch('requests.Session.get', autospec=True, side_effect=checked_get): + with mock.patch('requests.Session.put', autospec=True, side_effect=checked_put): + yield + + +@responses.activate +@pytest.mark.parametrize(('breakage', 'error'), [ + (None, None), + ('missing_cert', 'Cannot find certificate file'), + ('missing_key', 'Cannot find key file'), + ('missing_cert_and_key', 'Wrong/missing cert'), +]) +def test_skopeo_copy_cert(breakage, error): + """ + Test authentication with a certificate + """ + runner = testing.CliRunner() + + certdir1 = tempfile.mkdtemp() + certdir2 = tempfile.mkdtemp() + try: + certs = {} + for certdir, reg in ((certdir1, 'registry1.example.com'), + (certdir2, 'registry2.example.com')): + cert = os.path.join(certdir, reg + '.cert') + if breakage not in ('missing_cert', 'missing_cert_and_key'): + with open(cert, 'w'): + pass + key = os.path.join(certdir, reg + '.key') + if breakage not in ('missing_key', 'missing_cert_and_key'): + with open(key, 'w'): + pass + certs[reg] = (cert, key) + + content_type = MEDIA_TYPE_MANIFEST_V2 + reg1 = MockRegistry('registry1.example.com') + reg2 = MockRegistry('registry2.example.com') + digest = reg1.add_fake_image('repo1', 'latest', content_type) + + with check_certificates(get_cert=certs['registry1.example.com'], + put_cert=certs['registry2.example.com']): + args = ['--src-cert-dir', certdir1, + '--dest-cert-dir', certdir2, + 'docker:registry1.example.com/repo1:latest', + 'docker:registry2.example.com/repo2:latest'] + + if breakage is None: + result = runner.invoke( + skopeo_lite.copy, + args, + catch_exceptions=False) + + assert result.exit_code == 0 + reg2.check_fake_image('repo2', 'latest', digest, content_type) + else: + with pytest.raises(Exception) as e: + runner.invoke( + skopeo_lite.copy, + args, + catch_exceptions=False) + assert error in str(e) + finally: + shutil.rmtree(certdir1) + shutil.rmtree(certdir2) + + +@contextmanager +def mock_system_certs(): + old_isdir = os.path.isdir + old_listdir = os.listdir + old_exists = os.path.exists + + def isdir(path): + if path.startswith('/etc/'): + return path in ('/etc/docker/certs.d/registry1.example.com', + '/etc/docker/certs.d/registry2.example.com') + else: + return old_isdir(path) + + def listdir(path): + if path.startswith('/etc/'): + if path in ('/etc/docker/certs.d/registry1.example.com', + '/etc/docker/certs.d/registry2.example.com'): + return ('client.cert', 'client.key') + else: + return None + else: + return old_listdir(path) + + def exists(path): + if path.startswith('/etc/'): + return path in ('/etc/docker/certs.d/registry1.example.com/client.cert', + '/etc/docker/certs.d/registry1.example.com/client.key', + '/etc/docker/certs.d/registry2.example.com/client.cert', + '/etc/docker/certs.d/registry2.example.com/client.key') + else: + return old_exists(path) + + with mock.patch('os.path.isdir', side_effect=isdir): + with mock.patch('os.listdir', side_effect=listdir): + with mock.patch('os.path.exists', side_effect=exists): + yield + + +@responses.activate +def test_skopeo_copy_system_cert(): + """ + Test using a certificate from a system directory + """ + runner = testing.CliRunner() + + reg1 = MockRegistry('registry1.example.com') + MockRegistry('registry2.example.com') + reg1.add_fake_image('repo1', 'latest', MEDIA_TYPE_MANIFEST_V2) + + with mock_system_certs(): + with check_certificates(get_cert=('/etc/docker/certs.d/registry1.example.com/client.cert', + '/etc/docker/certs.d/registry1.example.com/client.key'), + put_cert=('/etc/docker/certs.d/registry2.example.com/client.cert', + '/etc/docker/certs.d/registry2.example.com/client.key')): + runner.invoke( + skopeo_lite.copy, + ['docker:registry1.example.com/repo1:latest', + 'docker:registry2.example.com/repo2:latest'], + catch_exceptions=False) + + +@responses.activate +@pytest.mark.parametrize(('src', 'dest', 'error'), [ + ('docker:registry1.example.com/repo1:latest', 'badtype:registry2.example.com/repo2:latest', + 'Unknown source/destination'), + ('docker:registry1.example.com/repo1:latest', 'docker:registry2.example.com', + 'Registry specification should be docker:REGISTRY/PATH') +]) +def test_skopeo_copy_cli_errors(src, dest, error): + pass + """ + Test errors triggered from bad command line arguments + """ + runner = testing.CliRunner() + + options = [] + + result = runner.invoke( + skopeo_lite.copy, + options + [src, dest]) + + assert error in result.output + assert result.exit_code != 0 + + +@responses.activate +@pytest.mark.parametrize(('src', 'dest', 'flags1', 'flags2', 'error'), [ + ('docker:registry1.example.com/repo1:latest', 'docker:registry2.example.com/repo2:latest', + '', 'bad_put_status', + 'Unexpected successful response'), + ('docker:registry1.example.com/repo1:latest', 'docker:registry2.example.com/repo2:latest', + '', 'bad_post_status', + 'Unexpected successful response'), + ('docker:registry1.example.com/repo1:latest', 'docker:registry1.example.com/repo2:latest', + 'bad_mount_status', '', + 'Blob mount had unexpected status'), + ('docker:registry1.example.com/repo1:latest', 'docker:registry2.example.com/repo2:latest', + 'bad_index_content_type', '', + 'Unhandled media type'), + ('docker:registry1.example.com/repo1:latest', 'docker:registry2.example.com/repo2:latest', + 'bad_content_type', '', + 'Unhandled media type'), + ('docker:registry1.example.com/repo1:latest', 'docker:registry2.example.com/repo2:latest', + '', 'include_query_parameters', + None), +]) +def test_skopeo_copy_protocol(src, dest, flags1, flags2, error): + pass + """ + Tests various error and other code paths related to variations in server responses; the + flags argument to the MockRegistry constructor is used to modify server behavior. + """ + runner = testing.CliRunner() + + content_type = MEDIA_TYPE_OCI_INDEX + + reg1 = MockRegistry('registry1.example.com', flags=flags1) + MockRegistry('registry2.example.com', flags=flags2) + reg1.add_fake_image('repo1', 'latest', content_type) + + if error is not None: + with pytest.raises(Exception) as e: + runner.invoke( + skopeo_lite.copy, + [src, dest], + catch_exceptions=False) + + assert error in str(e) + else: + result = runner.invoke( + skopeo_lite.copy, + [src, dest]) + + assert result.exit_code == 0 diff --git a/production.ini b/production.ini index af62890f1d..8f75d64f5a 100644 --- a/production.ini +++ b/production.ini @@ -173,6 +173,13 @@ use = egg:bodhi-server # The skopeo executable to use to copy container images. # You can put credentials for skopeo to use in $HOME/.docker/config.json # https://github.com/projectatomic/skopeo#private-registries-with-authentication +# +# An alternative command is bodhi-skopeo-lite, installed by bodhi. It has various limitations +# (including not reading $HOME/.docker/config.json - it supports certificate authentication +# and credentials passed on the command line), but supports manifest lists and OCI image indexes, +# allowing copying multi-arch containers. See https://github.com/containers/image/pull/400 for +# work to add such support to skopeo proper. +# # skopeo.cmd = /usr/bin/skopeo # Comma separated list of extra flags to pass to the skopeo copy command. diff --git a/setup.py b/setup.py index f9788cf7cb..ab4801530c 100644 --- a/setup.py +++ b/setup.py @@ -162,6 +162,7 @@ def get_requirements(requirements_file='requirements.txt'): bodhi-approve-testing = bodhi.server.scripts.approve_testing:main bodhi-manage-releases = bodhi.server.scripts.manage_releases:main bodhi-check-policies = bodhi.server.scripts.check_policies:check + bodhi-skopeo-lite = bodhi.server.scripts.skopeo_lite:main [moksha.consumer] masher = bodhi.server.consumers.masher:Masher updates = bodhi.server.consumers.updates:UpdatesHandler