From 18761fa1518ef6f22e0361d8b88418960832896d Mon Sep 17 00:00:00 2001 From: Randy Barlow Date: Thu, 15 Feb 2018 15:41:10 -0500 Subject: [PATCH] Add support for composing containers. fixes #2028 Signed-off-by: Randy Barlow --- bodhi/server/config.py | 14 +++++++++++ bodhi/server/consumers/masher.py | 40 +++++++++++++++++++++++++++++++- bodhi/server/push.py | 32 ++++++++++++++++--------- docs/user/release_notes.rst | 1 + production.ini | 14 +++++++++++ 5 files changed, 89 insertions(+), 12 deletions(-) diff --git a/bodhi/server/config.py b/bodhi/server/config.py index 8d862d2b15..0611145bfd 100644 --- a/bodhi/server/config.py +++ b/bodhi/server/config.py @@ -358,6 +358,12 @@ class BodhiConfig(dict): 'captcha.ttl': { 'value': 300, 'validator': int}, + 'container.destination_registry': { + 'value': 'registry.fedoraproject.org', + 'validator': six.text_type}, + 'container.source_registry': { + 'value': 'candidate-registry.fedoraproject.org', + 'validator': six.text_type}, 'cors_connect_src': { 'value': 'https://*.fedoraproject.org/ wss://hub.fedoraproject.org:9939/', 'validator': six.text_type}, @@ -547,6 +553,14 @@ class BodhiConfig(dict): 'site_requirements': { 'value': 'dist.rpmdeplint dist.upgradepath', 'validator': six.text_type}, + 'skopeo.cmd': { + 'value': '/usr/bin/skopeo', + 'validator': six.text_type, + }, + 'skopeo.extra_copy_flags': { + 'value': '', + 'validator': six.text_type, + }, 'smtp_server': { 'value': None, 'validator': _validate_none_or(six.text_type)}, diff --git a/bodhi/server/consumers/masher.py b/bodhi/server/consumers/masher.py index 227d4e2910..5c193869ec 100644 --- a/bodhi/server/consumers/masher.py +++ b/bodhi/server/consumers/masher.py @@ -304,7 +304,7 @@ def get_masher(content_type): ComposerThread or None: Either a ContainerComposerThread, RPMComposerThread, or a ModuleComposerThread, as appropriate, or None if no masher is found. """ - mashers = [RPMComposerThread, ModuleComposerThread] + mashers = [ContainerComposerThread, RPMComposerThread, ModuleComposerThread] for possible in mashers: if possible.ctype is content_type: return possible @@ -859,6 +859,44 @@ def sort_by_days_in_testing(self, updates): return updates +class ContainerComposerThread(ComposerThread): + """Use skopeo to copy and tag container images.""" + + ctype = ContentType.container + + def _compose_updates(self): + """Use skopeo to copy images to the correct repos and tags.""" + source_registry = config['container.source_registry'] + destination_registry = config['container.destination_registry'] + + for update in self.compose.updates: + + if update.request is UpdateRequest.stable: + destination_tag = 'latest' + else: + destination_tag = 'testing' + + for build in update.builds: + image_name = '{}/{}'.format(build.release.branch, build.package.name) + name, version, release = build.nvr.rsplit('-', 2) + version_release = '{}-{}'.format(version, release) + for dtag in [version_release, version, destination_tag]: + source_url = 'docker://{}/{}:{}'.format(source_registry, image_name, + version_release) + destination_url = 'docker://{}/{}:{}'.format(destination_registry, image_name, + dtag) + skopeo_cmd = [ + config.get('skopeo.cmd'), 'copy', config.get('skopeo.extra_copy_flags'), + source_url, destination_url] + log.info(skopeo_cmd) + skopeo_process = subprocess.Popen( + skopeo_cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = skopeo_process.communicate() + + if skopeo_process.returncode: + raise RuntimeError(out + err) + + class PungiComposerThread(ComposerThread): """Compose update with Pungi.""" diff --git a/bodhi/server/push.py b/bodhi/server/push.py index 8c33d14872..5a350c0a0a 100644 --- a/bodhi/server/push.py +++ b/bodhi/server/push.py @@ -160,17 +160,27 @@ def push(username, cert_prefix, **kwargs): click.echo('\nSending masher.start fedmsg') # Because we're a script, we want to send to the fedmsg-relay, # that's why we say active=True - bodhi.server.notifications.init(active=True, cert_prefix=cert_prefix) - bodhi.server.notifications.publish( - topic='masher.start', - msg=dict( - api_version=2, - composes=composes, - resume=resume, - agent=username, - ), - force=True, - ) + import mock + from bodhi.server.consumers import masher + from pyramid.paster import setup_logging + setup_logging('/home/vagrant/bodhi/development.ini') + with mock.patch('bodhi.server.consumers.masher.fedmsg.consumers.FedmsgConsumer.__init__'): + m = masher.Masher(mock.MagicMock()) + import logging + m.log = logging.getLogger('bowlofeggs') + m.work({'body': {'msg': dict(api_version=2, composes=composes, resume=resume, + agent=username)}}) + #bodhi.server.notifications.init(active=True, cert_prefix=cert_prefix) + #bodhi.server.notifications.publish( + # topic='masher.start', + # msg=dict( + # api_version=2, + # composes=composes, + # resume=resume, + # agent=username, + # ), + # force=True, + #) def _filter_releases(session, query, releases=None): diff --git a/docs/user/release_notes.rst b/docs/user/release_notes.rst index 342faca558..0ab8684a3c 100644 --- a/docs/user/release_notes.rst +++ b/docs/user/release_notes.rst @@ -9,6 +9,7 @@ Dependency changes ^^^^^^^^^^^^^^^^^^ * Pungi 4.1.20 or higher is now required. +* Skopeo is now a required dependency for Bodhi installations that compose containers. Bugs diff --git a/production.ini b/production.ini index d31b32813d..695781b63b 100644 --- a/production.ini +++ b/production.ini @@ -156,6 +156,20 @@ use = egg:bodhi-server # What to pass to Pungi's --label flag, which is metadata included in its composeinfo.json. # pungi.labeltype = Update +# The skopeo executable to use to copy container images. +# You can put credentials for skopeo to use in $HOME/.docker/config.json +# https://github.com/projectatomic/skopeo#private-registries-with-authentication +# skopeo.cmd = /usr/bin/skopeo + +# Extra flags to pass to the skopeo copy command. +# During development, it can be handy to set this to --dest-tls-verify=false if you want to push +# into a local registry. +# skopeo.extra_copy_flags = + +# Container hostnames. You can specify a port as well, using the traditional syntax (i.e., localhost:5000). +# container.destination_registry = registry.fedoraproject.org +# container.source_registry = candidate-registry.fedoraproject.org + ## ## Mirror settings