From 3b91dc4bcae5dc3104f91ca5f51a6aa03f7bba68 Mon Sep 17 00:00:00 2001 From: Randy Barlow Date: Wed, 14 Feb 2018 16:01:34 -0500 Subject: [PATCH] Add support for composing container updates. fixes #2028 Signed-off-by: Randy Barlow --- bodhi/server/consumers/masher.py | 851 ++++++++++---------- bodhi/tests/server/consumers/test_masher.py | 698 +++++++++------- 2 files changed, 859 insertions(+), 690 deletions(-) diff --git a/bodhi/server/consumers/masher.py b/bodhi/server/consumers/masher.py index 5ce781c4d8..1cb1613198 100644 --- a/bodhi/server/consumers/masher.py +++ b/bodhi/server/consumers/masher.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright © 2007-2017 Red Hat, Inc. and others. +# Copyright © 2007-2018 Red Hat, Inc. and others. # # This file is part of Bodhi. # @@ -296,29 +296,28 @@ def work(self, msg): def get_masher(content_type): """ - Return the correct MasherThread subclass for content_type. + Return the correct ComposerThread subclass for content_type. Args: content_type (bodhi.server.models.EnumSymbol): The content type we seek a masher for. Return: - MasherThread or None: Either an RPMMasherThread or a ModuleMasherThread, as appropriate, or - None if no masher is found. + ComposerThread or None: Either a ContainerComposerThread, RPMComposerThread, or a + ModuleComposerThread, as appropriate, or None if no masher is found. """ - mashers = [RPMMasherThread, ModuleMasherThread] + mashers = [ContainerComposerThread, RPMComposerThread, ModuleComposerThread] for possible in mashers: if possible.ctype is content_type: return possible -class MasherThread(threading.Thread): - """The base class that defines common things for all mashings.""" +class ComposerThread(threading.Thread): + """The base class that defines common things for all composes.""" ctype = None - pungi_template_config_key = None def __init__(self, compose, agent, log, db_factory, mash_dir, resume=False): """ - Initialize the MasherThread. + Initialize the ComposerThread. Args: compose (dict): A dictionary representation of the Compose to run, formatted like the @@ -330,11 +329,10 @@ def __init__(self, compose, agent, log, db_factory, mash_dir, resume=False): mash_dir (basestring): A path to a directory to generate the mash in. resume (bool): Whether or not we are resuming a previous failed mash. Defaults to False. """ - super(MasherThread, self).__init__() + super(ComposerThread, self).__init__() self.db_factory = db_factory self.log = log self.agent = agent - self.mash_dir = mash_dir self._compose = compose self.resume = resume self.add_tags_async = [] @@ -342,10 +340,7 @@ def __init__(self, compose, agent, log, db_factory, mash_dir, resume=False): self.add_tags_sync = [] self.move_tags_sync = [] self.testing_digest = {} - self.path = None self.success = False - self.devnull = None - self._startyear = None def run(self): """Run the thread by managing a db transaction and calling work().""" @@ -365,7 +360,7 @@ def run(self): self.compose.error_message = unicode(e) self.save_state(ComposeState.failed) - self.log.exception('MasherThread failed. Transaction rolled back.') + self.log.exception('ComposerThread failed. Transaction rolled back.') finally: self.compose = None self.db = None @@ -394,13 +389,12 @@ def work(self): # tasks for testing updates. For stable updates, we should just add the # dist_tag and do everything else other than mashing/updateinfo, since # the nightly build-branched cron job mashes for us. - self.skip_mash = False + self.skip_compose = False if (self.compose.release.state is ReleaseState.pending and self.compose.request is UpdateRequest.stable): - self.skip_mash = True + self.skip_compose = True - self.log.info('Running MasherThread(%s)' % self.id) - self.init_state() + self.log.info('Running ComposerThread(%s)' % self.id) notifications.publish( topic="mashtask.mashing", @@ -427,25 +421,7 @@ def work(self): self.expire_buildroot_overrides() self.remove_pending_tags() - if not self.skip_mash: - mash_process = self.mash() - - # Things we can do while we're mashing - self.generate_testing_digest() - - if not self.skip_mash: - uinfo = self.generate_updateinfo() - - self.wait_for_mash(mash_process) - - uinfo.insert_updateinfo(self.path) - - if not self.skip_mash: - self.sanity_check_repo() - self.stage_repo() - - # Wait for the repo to hit the master mirror - self.wait_for_sync() + self._compose_updates() self._mark_status_changes() self.save_state(ComposeState.notifying) @@ -474,7 +450,7 @@ def work(self): self.remove_state() except Exception: - self.log.exception('Exception in MasherThread(%s)' % self.id) + self.log.exception('Exception in ComposerThread(%s)' % self.id) self.save_state() raise finally: @@ -540,12 +516,6 @@ def eject_from_mash(self, update, reason): force=True, ) - def init_state(self): - """Create the mash_dir if it doesn't exist.""" - if not os.path.exists(self.mash_dir): - self.log.info('Creating %s' % self.mash_dir) - os.makedirs(self.mash_dir) - def save_state(self, state=None): """ Save the state of this push so it can be resumed later if necessary. @@ -565,14 +535,9 @@ def load_state(self): self._checkpoints = json.loads(self.compose.checkpoints) self.log.info('Masher state loaded from %s', self.compose) self.log.info(self.compose.state) - if 'completed_repo' in self._checkpoints: - self.path = self._checkpoints['completed_repo'] - self.log.info('Resuming push with completed repo: %s' % self.path) - return - self.log.info('Resuming push without any completed repos') def remove_state(self): - """Remove the mash lock file.""" + """Remove the Compose object from the database.""" self.log.info('Removing state: %s', self.compose) self.db.delete(self.compose) @@ -583,10 +548,6 @@ def finish(self, success): Args: success (bool): True if the mash had been successful, False otherwise. """ - if hasattr(self, '_pungi_conf_dir') and os.path.exists(self._pungi_conf_dir) and success: - # Let's clean up the pungi configs we wrote - shutil.rmtree(self._pungi_conf_dir) - self.log.info('Thread(%s) finished. Success: %r' % (self.id, success)) notifications.publish( topic="mashtask.complete", @@ -634,7 +595,7 @@ def _determine_tag_actions(self): self.eject_from_mash(update, reason) break - if self.skip_mash: + if self.skip_compose: add_tags.append((update.requested_tag, build.nvr)) else: move_tags.append((from_tag, update.requested_tag, @@ -700,148 +661,6 @@ def remove_pending_tags(self): self.log.debug('remove_pending_tags koji.multiCall result = %r', result) - def copy_additional_pungi_files(self, pungi_conf_dir, template_env): - """ - Child classes should override this to place type-specific Pungi files in the config dir. - - Args: - pungi_conf_dir (basestring): A path to the directory that Pungi's configs are being - written to. - template_env (jinja2.Environment): The jinja2 environment to be used while rendering the - variants.xml template. - raises: - NotImplementedError: The parent class does not implement this method. - """ - raise NotImplementedError - - def create_pungi_config(self): - """Create a temp dir and render the Pungi config templates into the dir.""" - loader = jinja2.FileSystemLoader(searchpath=config.get('pungi.basepath')) - env = jinja2.Environment(loader=loader, - autoescape=False, - block_start_string='[%', - block_end_string='%]', - variable_start_string='[[', - variable_end_string=']]', - comment_start_string='[#', - comment_end_string='#]') - - env.globals['id'] = self.id - env.globals['release'] = self.compose.release - env.globals['request'] = self.compose.request - env.globals['updates'] = self.compose.updates - - config_template = config.get(self.pungi_template_config_key) - template = env.get_template(config_template) - - self._pungi_conf_dir = tempfile.mkdtemp(prefix='bodhi-pungi-%s-' % self.id) - - with open(os.path.join(self._pungi_conf_dir, 'pungi.conf'), 'w') as conffile: - conffile.write(template.render()) - - self.copy_additional_pungi_files(self._pungi_conf_dir, env) - - def mash(self): - """ - Launch the Pungi child process to "punge" the repository. - - Returns: - subprocess.Popen: A process handle to the child Pungi process. - Raises: - Exception: If the child Pungi process exited with a non-0 exit code within 3 seconds. - """ - if self.path: - self.log.info('Skipping completed repo: %s', self.path) - return - - # We have a thread-local devnull FD so that we can close them after the mash is done - self.devnull = open(os.devnull, 'wb') - - self.create_pungi_config() - config_file = os.path.join(self._pungi_conf_dir, 'pungi.conf') - self._label = '%s-%s' % (config.get('pungi.labeltype'), - datetime.utcnow().strftime('%Y%m%d.%H%M')) - pungi_cmd = [config.get('pungi.cmd'), - '--config', config_file, - '--quiet', - '--target-dir', self.mash_dir, - '--old-composes', self.mash_dir, - '--no-latest-link', - '--label', self._label] - pungi_cmd += config.get('pungi.extracmdline') - - self.log.info('Running the pungi command: %s', pungi_cmd) - mash_process = subprocess.Popen(pungi_cmd, - # Nope. No shell for you - shell=False, - # Should be useless, but just to set something predictable - cwd=self.mash_dir, - # Pungi will logs its stdout into pungi.global.log - stdout=self.devnull, - # Stderr should also go to pungi.global.log if it starts - stderr=subprocess.PIPE, - # We will never have additional input - stdin=self.devnull) - self.log.info('Pungi running as PID: %s', mash_process.pid) - # Since the mash process takes a long time, we can safely just wait 3 seconds to abort the - # entire mash early if Pungi fails to start up correctly. - time.sleep(3) - if mash_process.poll() not in [0, None]: - self.log.error('Pungi process terminated with error within 3 seconds! Abandoning!') - _, err = mash_process.communicate() - self.log.error('Stderr: %s', err) - self.devnull.close() - raise Exception('Pungi returned error, aborting!') - - # This is used to find the generated directory post-mash. - # This is stored at the time of start so that even if the update run crosses the year - # border, we can still find it back. - self._startyear = datetime.utcnow().year - - return mash_process - - def wait_for_mash(self, mash_process): - """ - Wait for the pungi process to exit and find the path of the repository that it produced. - - Args: - mash_process (subprocess.Popen): The Popen handle of the running child process. - Raises: - Exception: If pungi's exit code is not 0, or if it is unable to find the directory that - Pungi created. - """ - self.save_state(ComposeState.punging) - if mash_process is None: - self.log.info('Not waiting for mash thread, as there was no mash') - return - self.log.info('Waiting for mash thread to finish') - _, err = mash_process.communicate() - self.devnull.close() - if mash_process.returncode != 0: - self.log.error('Mashing process exited with exit code %d', mash_process.returncode) - self.log.error('Stderr: %s', err) - raise Exception('Pungi exited with status %d' % mash_process.returncode) - else: - self.log.info('Mashing finished') - - # Find the path Pungi just created - requesttype = 'updates' - if self.compose.request is UpdateRequest.testing: - requesttype = 'updates-testing' - # The year here is used so that we can correctly find the latest updates mash, so that we - # find updates-20420101.1 instead of updates-testing-20420506.5 - prefix = '%s-%d-%s-%s*' % (self.compose.release.id_prefix.title(), - int(self.compose.release.version), - requesttype, - self._startyear) - - paths = sorted(glob.glob(os.path.join(self.mash_dir, prefix))) - if len(paths) < 1: - raise Exception('We were unable to find a path with prefix %s in mashdir' % prefix) - self.log.debug('Paths: %s', paths) - self.path = paths[-1] - self._checkpoints['completed_repo'] = self.path - def _mark_status_changes(self): """Mark each update's status as fulfilling its request.""" self.log.info('Updating update statuses.') @@ -886,217 +705,73 @@ def generate_testing_digest(self): self.add_to_digest(update) self.log.info('Testing digest generation for %s complete' % self.compose.release.name) - def generate_updateinfo(self): - """ - Create the updateinfo.xml file for this repository. + def send_notifications(self): + """Send fedmsgs to announce completion of mashing for each update.""" + self.log.info('Sending notifications') + try: + agent = os.getlogin() + except OSError: # this can happen when building on koji + agent = u'masher' + for update in self.compose.updates: + topic = u'update.complete.%s' % update.request + notifications.publish( + topic=topic, + msg=dict(update=update, agent=agent), + force=True, + ) - Returns: - bodhi.server.metadata.UpdateInfoMetadata: The updateinfo model that was created for this - repository. - """ - self.log.info('Generating updateinfo for %s' % self.compose.release.name) - self.save_state(ComposeState.updateinfo) - uinfo = UpdateInfoMetadata(self.compose.release, self.compose.request, - self.db, self.mash_dir) - self.log.info('Updateinfo generation for %s complete' % self.compose.release.name) - return uinfo + @checkpoint + def modify_bugs(self): + """Mark bugs on each Update as modified.""" + self.log.info('Updating bugs') + for update in self.compose.updates: + self.log.debug('Modifying bugs for %s', update.title) + update.modify_bugs() - def sanity_check_repo(self): - """Sanity check our repo. + @checkpoint + def status_comments(self): + """Add bodhi system comments to each update.""" + self.log.info('Commenting on updates') + for update in self.compose.updates: + update.status_comment(self.db) - - make sure we didn't compose a repo full of symlinks - - sanity check our repodata + @checkpoint + def send_stable_announcements(self): + """Send the stable announcement e-mails out.""" + self.log.info('Sending stable update announcements') + for update in self.compose.updates: + if update.request is UpdateRequest.stable: + update.send_update_notice() - This basically checks that pungi was run with gather_method='hardlink-or-copy' so that - we get a repository with either hardlinks or copied files. - This means that we when we go and sync generated repositories out, we do not need to take - special case to copy the target files rather than symlinks. - """ - self.log.info("Running sanity checks on %s" % self.path) + @checkpoint + def send_testing_digest(self): + """Send digest mail to mailing lists.""" + self.log.info('Sending updates-testing digest') + sechead = u'The following %s Security updates need testing:\n Age URL\n' + crithead = u'The following %s Critical Path updates have yet to be approved:\n Age URL\n' + testhead = u'The following builds have been pushed to %s updates-testing\n\n' - arches = os.listdir(os.path.join(self.path, 'compose', 'Everything')) - for arch in arches: - # sanity check our repodata - try: - if arch == 'source': - repodata = os.path.join(self.path, 'compose', - 'Everything', arch, 'tree', 'repodata') - else: - repodata = os.path.join(self.path, 'compose', - 'Everything', arch, 'os', 'repodata') - sanity_check_repodata(repodata) - except Exception: - self.log.exception("Repodata sanity check failed!") - raise + for prefix, content in six.iteritems(self.testing_digest): + release = self.db.query(Release).filter_by(long_name=prefix).one() + test_list_key = '%s_test_announce_list' % ( + release.id_prefix.lower().replace('-', '_')) + test_list = config.get(test_list_key) + if not test_list: + log.warn('%r undefined. Not sending updates-testing digest', + test_list_key) + continue - # make sure that pungi didn't symlink our packages - try: - if arch == 'source': - dirs = [('tree', 'Packages')] - else: - dirs = [('debug', 'tree', 'Packages'), ('os', 'Packages')] - - # Example of full path we are checking: - # self.path/compose/Everything/os/Packages/s/something.rpm - for checkdir in dirs: - checkdir = os.path.join(self.path, 'compose', 'Everything', arch, *checkdir) - subdirs = os.listdir(checkdir) - # subdirs is the self.path/compose/Everything/os/Packages/{a,b,c,...}/ dirs - # - # Let's check the first file in each subdir. If they are correct, we'll assume - # the rest is correct - # This is to avoid tons and tons of IOPS for a bunch of files put in in the - # same way - for subdir in subdirs: - for checkfile in os.listdir(os.path.join(checkdir, subdir)): - if not checkfile.endswith('.rpm'): - continue - if os.path.islink(os.path.join(checkdir, subdir, checkfile)): - self.log.error('Pungi out directory contains at least one ' - 'symlink at %s', checkfile) - raise Exception('Symlinks found') - # We have checked the first rpm in the subdir - break - except Exception: - self.log.exception('Unable to check pungi mashed repositories') - raise - - return True - - def stage_repo(self): - """Symlink our updates repository into the staging directory.""" - stage_dir = config.get('mash_stage_dir') - if not os.path.isdir(stage_dir): - self.log.info('Creating mash_stage_dir %s', stage_dir) - os.mkdir(stage_dir) - link = os.path.join(stage_dir, self.id) - if os.path.islink(link): - os.unlink(link) - self.log.info("Creating symlink: %s => %s" % (link, self.path)) - os.symlink(self.path, link) - - def wait_for_sync(self): - """ - Block until our repomd.xml hits the master mirror. - - Raises: - Exception: If no folder other than "source" was found in the mash_path. - """ - self.log.info('Waiting for updates to hit the master mirror') - notifications.publish( - topic="mashtask.sync.wait", - msg=dict(repo=self.id, agent=self.agent), - force=True, - ) - mash_path = os.path.join(self.path, 'compose', 'Everything') - checkarch = None - # Find the first non-source arch to check against - for arch in os.listdir(mash_path): - if arch == 'source': - continue - checkarch = arch - break - if not checkarch: - raise Exception('Not found an arch to wait_for_sync with') - - repomd = os.path.join(mash_path, arch, 'os', 'repodata', 'repomd.xml') - if not os.path.exists(repomd): - self.log.error('Cannot find local repomd: %s', repomd) - return - - master_repomd_url = self._get_master_repomd_url(arch) - - with open(repomd) as repomdf: - checksum = hashlib.sha1(repomdf.read()).hexdigest() - while True: - try: - self.log.info('Polling %s' % master_repomd_url) - masterrepomd = urllib2.urlopen(master_repomd_url) - except (urllib2.URLError, urllib2.HTTPError): - self.log.exception('Error fetching repomd.xml') - time.sleep(200) - continue - newsum = hashlib.sha1(masterrepomd.read()).hexdigest() - if newsum == checksum: - self.log.info("master repomd.xml matches!") - notifications.publish( - topic="mashtask.sync.done", - msg=dict(repo=self.id, agent=self.agent), - force=True, - ) - return - - self.log.debug("master repomd.xml doesn't match! %s != %s for %r", - checksum, newsum, self.id) - time.sleep(200) - - def send_notifications(self): - """Send fedmsgs to announce completion of mashing for each update.""" - self.log.info('Sending notifications') - try: - agent = os.getlogin() - except OSError: # this can happen when building on koji - agent = u'masher' - for update in self.compose.updates: - topic = u'update.complete.%s' % update.request - notifications.publish( - topic=topic, - msg=dict(update=update, agent=agent), - force=True, - ) - - @checkpoint - def modify_bugs(self): - """Mark bugs on each Update as modified.""" - self.log.info('Updating bugs') - for update in self.compose.updates: - self.log.debug('Modifying bugs for %s', update.title) - update.modify_bugs() - - @checkpoint - def status_comments(self): - """Add bodhi system comments to each update.""" - self.log.info('Commenting on updates') - for update in self.compose.updates: - update.status_comment(self.db) - - @checkpoint - def send_stable_announcements(self): - """Send the stable announcement e-mails out.""" - self.log.info('Sending stable update announcements') - for update in self.compose.updates: - if update.request is UpdateRequest.stable: - update.send_update_notice() - - @checkpoint - def send_testing_digest(self): - """Send digest mail to mailing lists.""" - self.log.info('Sending updates-testing digest') - sechead = u'The following %s Security updates need testing:\n Age URL\n' - crithead = u'The following %s Critical Path updates have yet to be approved:\n Age URL\n' - testhead = u'The following builds have been pushed to %s updates-testing\n\n' - - for prefix, content in six.iteritems(self.testing_digest): - release = self.db.query(Release).filter_by(long_name=prefix).one() - test_list_key = '%s_test_announce_list' % ( - release.id_prefix.lower().replace('-', '_')) - test_list = config.get(test_list_key) - if not test_list: - log.warn('%r undefined. Not sending updates-testing digest', - test_list_key) - continue - - log.debug("Sending digest for updates-testing %s" % prefix) - maildata = u'' - security_updates = self.get_security_updates(prefix) - if security_updates: - maildata += sechead % prefix - for update in security_updates: - maildata += u' %3i %s %s\n' % ( - update.days_in_testing, - update.abs_url(), - update.title) - maildata += '\n\n' + log.debug("Sending digest for updates-testing %s" % prefix) + maildata = u'' + security_updates = self.get_security_updates(prefix) + if security_updates: + maildata += sechead % prefix + for update in security_updates: + maildata += u' %3i %s %s\n' % ( + update.days_in_testing, + update.abs_url(), + update.title) + maildata += '\n\n' critpath_updates = self.get_unapproved_critpath_updates(prefix) if critpath_updates: @@ -1175,12 +850,142 @@ def sort_by_days_in_testing(self, updates): updates.sort(key=lambda update: update.days_in_testing, reverse=True) return updates + +class PungiComposerThread(ComposerThread): + """Compose update with Pungi.""" + + pungi_template_config_key = None + + def __init__(self, compose, agent, log, db_factory, mash_dir, resume=False): + """ + Initialize the ComposerThread. + + Args: + compose (dict): A dictionary representation of the Compose to run, formatted like the + output of :meth:`Compose.__json__`. + agent (basestring): The user who is executing the mash. + log (logging.Logger): A logger to use for this mash. + db_factory (bodhi.server.util.TransactionalSessionMaker): A DB session to use while + mashing. + mash_dir (basestring): A path to a directory to generate the mash in. + resume (bool): Whether or not we are resuming a previous failed mash. Defaults to False. + """ + super(PungiComposerThread, self).__init__(compose, agent, log, db_factory, mash_dir, resume) + self.devnull = None + self.mash_dir = mash_dir + self.path = None + self._startyear = None + + def finish(self, success): + """ + Clean up pungi configs if the mash was successful, and send logs and fedmsgs. + + Args: + success (bool): True if the mash had been successful, False otherwise. + """ + if hasattr(self, '_pungi_conf_dir') and os.path.exists(self._pungi_conf_dir) and success: + # Let's clean up the pungi configs we wrote + shutil.rmtree(self._pungi_conf_dir) + + # The superclass will handle the logs and fedmsg. + super(PungiComposerThread, self).finish(success) + + def load_state(self): + """Set self.path if completed_repo is found in checkpoints.""" + super(PungiComposerThread, self).load_state() + if 'completed_repo' in self._checkpoints: + self.path = self._checkpoints['completed_repo'] + self.log.info('Resuming push with completed repo: %s' % self.path) + return + self.log.info('Resuming push without any completed repos') + + def _compose_updates(self): + """Start pungi, generate updateinfo, wait for pungi, and wait for the mirrors.""" + if not os.path.exists(self.mash_dir): + self.log.info('Creating %s' % self.mash_dir) + os.makedirs(self.mash_dir) + + if not self.skip_compose: + pungi_process = self._punge() + + # Things we can do while Pungi is running + self.generate_testing_digest() + + if not self.skip_compose: + uinfo = self._generate_updateinfo() + + self._wait_for_pungi(pungi_process) + + uinfo.insert_updateinfo(self.path) + + self._sanity_check_repo() + self._stage_repo() + + # Wait for the repo to hit the master mirror + self._wait_for_sync() + + def _copy_additional_pungi_files(self, pungi_conf_dir, template_env): + """ + Child classes should override this to place type-specific Pungi files in the config dir. + + Args: + pungi_conf_dir (basestring): A path to the directory that Pungi's configs are being + written to. + template_env (jinja2.Environment): The jinja2 environment to be used while rendering the + variants.xml template. + raises: + NotImplementedError: The parent class does not implement this method. + """ + raise NotImplementedError + + def _create_pungi_config(self): + """Create a temp dir and render the Pungi config templates into the dir.""" + loader = jinja2.FileSystemLoader(searchpath=config.get('pungi.basepath')) + env = jinja2.Environment(loader=loader, + autoescape=False, + block_start_string='[%', + block_end_string='%]', + variable_start_string='[[', + variable_end_string=']]', + comment_start_string='[#', + comment_end_string='#]') + + env.globals['id'] = self.id + env.globals['release'] = self.compose.release + env.globals['request'] = self.compose.request + env.globals['updates'] = self.compose.updates + + config_template = config.get(self.pungi_template_config_key) + template = env.get_template(config_template) + + self._pungi_conf_dir = tempfile.mkdtemp(prefix='bodhi-pungi-%s-' % self.id) + + with open(os.path.join(self._pungi_conf_dir, 'pungi.conf'), 'w') as conffile: + conffile.write(template.render()) + + self._copy_additional_pungi_files(self._pungi_conf_dir, env) + + def _generate_updateinfo(self): + """ + Create the updateinfo.xml file for this repository. + + Returns: + bodhi.server.metadata.UpdateInfoMetadata: The updateinfo model that was created for this + repository. + """ + self.log.info('Generating updateinfo for %s' % self.compose.release.name) + self.save_state(ComposeState.updateinfo) + uinfo = UpdateInfoMetadata(self.compose.release, self.compose.request, + self.db, self.mash_dir) + self.log.info('Updateinfo generation for %s complete' % self.compose.release.name) + return uinfo + def _get_master_repomd_url(self, arch): """ Return the master repomd URL for the given arch. Look up the correct *_master_repomd setting in the config and use it to form the URL that - wait_for_sync() will use to determine when the repository has been synchronized to the + _wait_for_sync() will use to determine when the repository has been synchronized to the master mirror. Args: @@ -1209,14 +1014,244 @@ def _get_master_repomd_url(self, arch): return master_repomd % (self.compose.release.version, arch) + def _punge(self): + """ + Launch the Pungi child process to "punge" the repository. + + Returns: + subprocess.Popen: A process handle to the child Pungi process. + Raises: + Exception: If the child Pungi process exited with a non-0 exit code within 3 seconds. + """ + if self.path: + self.log.info('Skipping completed repo: %s', self.path) + return + + # We have a thread-local devnull FD so that we can close them after the mash is done + self.devnull = open(os.devnull, 'wb') + + self._create_pungi_config() + config_file = os.path.join(self._pungi_conf_dir, 'pungi.conf') + self._label = '%s-%s' % (config.get('pungi.labeltype'), + datetime.utcnow().strftime('%Y%m%d.%H%M')) + pungi_cmd = [config.get('pungi.cmd'), + '--config', config_file, + '--quiet', + '--target-dir', self.mash_dir, + '--old-composes', self.mash_dir, + '--no-latest-link', + '--label', self._label] + pungi_cmd += config.get('pungi.extracmdline') + + self.log.info('Running the pungi command: %s', pungi_cmd) + mash_process = subprocess.Popen(pungi_cmd, + # Nope. No shell for you + shell=False, + # Should be useless, but just to set something predictable + cwd=self.mash_dir, + # Pungi will logs its stdout into pungi.global.log + stdout=self.devnull, + # Stderr should also go to pungi.global.log if it starts + stderr=subprocess.PIPE, + # We will never have additional input + stdin=self.devnull) + self.log.info('Pungi running as PID: %s', mash_process.pid) + # Since the mash process takes a long time, we can safely just wait 3 seconds to abort the + # entire mash early if Pungi fails to start up correctly. + time.sleep(3) + if mash_process.poll() not in [0, None]: + self.log.error('Pungi process terminated with error within 3 seconds! Abandoning!') + _, err = mash_process.communicate() + self.log.error('Stderr: %s', err) + self.devnull.close() + raise Exception('Pungi returned error, aborting!') + + # This is used to find the generated directory post-mash. + # This is stored at the time of start so that even if the update run crosses the year + # border, we can still find it back. + self._startyear = datetime.utcnow().year + + return mash_process + + def _sanity_check_repo(self): + """Sanity check our repo. + + - make sure we didn't compose a repo full of symlinks + - sanity check our repodata + + This basically checks that pungi was run with gather_method='hardlink-or-copy' so that + we get a repository with either hardlinks or copied files. + This means that we when we go and sync generated repositories out, we do not need to take + special case to copy the target files rather than symlinks. + """ + self.log.info("Running sanity checks on %s" % self.path) + + arches = os.listdir(os.path.join(self.path, 'compose', 'Everything')) + for arch in arches: + # sanity check our repodata + try: + if arch == 'source': + repodata = os.path.join(self.path, 'compose', + 'Everything', arch, 'tree', 'repodata') + else: + repodata = os.path.join(self.path, 'compose', + 'Everything', arch, 'os', 'repodata') + sanity_check_repodata(repodata) + except Exception: + self.log.exception("Repodata sanity check failed!") + raise + + # make sure that pungi didn't symlink our packages + try: + if arch == 'source': + dirs = [('tree', 'Packages')] + else: + dirs = [('debug', 'tree', 'Packages'), ('os', 'Packages')] + + # Example of full path we are checking: + # self.path/compose/Everything/os/Packages/s/something.rpm + for checkdir in dirs: + checkdir = os.path.join(self.path, 'compose', 'Everything', arch, *checkdir) + subdirs = os.listdir(checkdir) + # subdirs is the self.path/compose/Everything/os/Packages/{a,b,c,...}/ dirs + # + # Let's check the first file in each subdir. If they are correct, we'll assume + # the rest is correct + # This is to avoid tons and tons of IOPS for a bunch of files put in in the + # same way + for subdir in subdirs: + for checkfile in os.listdir(os.path.join(checkdir, subdir)): + if not checkfile.endswith('.rpm'): + continue + if os.path.islink(os.path.join(checkdir, subdir, checkfile)): + self.log.error('Pungi out directory contains at least one ' + 'symlink at %s', checkfile) + raise Exception('Symlinks found') + # We have checked the first rpm in the subdir + break + except Exception: + self.log.exception('Unable to check pungi mashed repositories') + raise + + return True + + def _stage_repo(self): + """Symlink our updates repository into the staging directory.""" + stage_dir = config.get('mash_stage_dir') + if not os.path.isdir(stage_dir): + self.log.info('Creating mash_stage_dir %s', stage_dir) + os.mkdir(stage_dir) + link = os.path.join(stage_dir, self.id) + if os.path.islink(link): + os.unlink(link) + self.log.info("Creating symlink: %s => %s" % (link, self.path)) + os.symlink(self.path, link) + + def _wait_for_pungi(self, pungi_process): + """ + Wait for the pungi process to exit and find the path of the repository that it produced. + + Args: + pungi_process (subprocess.Popen): The Popen handle of the running child process. + Raises: + Exception: If pungi's exit code is not 0, or if it is unable to find the directory that + Pungi created. + """ + self.save_state(ComposeState.punging) + if pungi_process is None: + self.log.info('Not waiting for pungi thread, as there was no pungi') + return + self.log.info('Waiting for pungi thread to finish') + _, err = pungi_process.communicate() + self.devnull.close() + if pungi_process.returncode != 0: + self.log.error('Pungi exited with exit code %d', pungi_process.returncode) + self.log.error('Stderr: %s', err) + raise Exception('Pungi exited with status %d' % pungi_process.returncode) + else: + self.log.info('Pungi finished') + + # Find the path Pungi just created + requesttype = 'updates' + if self.compose.request is UpdateRequest.testing: + requesttype = 'updates-testing' + # The year here is used so that we can correctly find the latest updates mash, so that we + # find updates-20420101.1 instead of updates-testing-20420506.5 + prefix = '%s-%d-%s-%s*' % (self.compose.release.id_prefix.title(), + int(self.compose.release.version), + requesttype, + self._startyear) + + paths = sorted(glob.glob(os.path.join(self.mash_dir, prefix))) + if len(paths) < 1: + raise Exception('We were unable to find a path with prefix %s in mashdir' % prefix) + self.log.debug('Paths: %s', paths) + self.path = paths[-1] + self._checkpoints['completed_repo'] = self.path + + def _wait_for_sync(self): + """ + Block until our repomd.xml hits the master mirror. + + Raises: + Exception: If no folder other than "source" was found in the mash_path. + """ + self.log.info('Waiting for updates to hit the master mirror') + notifications.publish( + topic="mashtask.sync.wait", + msg=dict(repo=self.id, agent=self.agent), + force=True, + ) + mash_path = os.path.join(self.path, 'compose', 'Everything') + checkarch = None + # Find the first non-source arch to check against + for arch in os.listdir(mash_path): + if arch == 'source': + continue + checkarch = arch + break + if not checkarch: + raise Exception('Not found an arch to _wait_for_sync with') + + repomd = os.path.join(mash_path, arch, 'os', 'repodata', 'repomd.xml') + if not os.path.exists(repomd): + self.log.error('Cannot find local repomd: %s', repomd) + return + + master_repomd_url = self._get_master_repomd_url(arch) + + with open(repomd) as repomdf: + checksum = hashlib.sha1(repomdf.read()).hexdigest() + while True: + try: + self.log.info('Polling %s' % master_repomd_url) + masterrepomd = urllib2.urlopen(master_repomd_url) + except (urllib2.URLError, urllib2.HTTPError): + self.log.exception('Error fetching repomd.xml') + time.sleep(200) + continue + newsum = hashlib.sha1(masterrepomd.read()).hexdigest() + if newsum == checksum: + self.log.info("master repomd.xml matches!") + notifications.publish( + topic="mashtask.sync.done", + msg=dict(repo=self.id, agent=self.agent), + force=True, + ) + return + + self.log.debug("master repomd.xml doesn't match! %s != %s for %r", + checksum, newsum, self.id) + time.sleep(200) + -class RPMMasherThread(MasherThread): +class RPMComposerThread(PungiComposerThread): """Run Pungi with configs that produce RPM repositories (yum/dnf and OSTrees).""" ctype = ContentType.rpm pungi_template_config_key = 'pungi.conf.rpm' - def copy_additional_pungi_files(self, pungi_conf_dir, template_env): + def _copy_additional_pungi_files(self, pungi_conf_dir, template_env): """ Generate and write the variants.xml file for this Pungi run. @@ -1232,13 +1267,13 @@ def copy_additional_pungi_files(self, pungi_conf_dir, template_env): variantsfile.write(variants_template.render()) -class ModuleMasherThread(MasherThread): +class ModuleComposerThread(PungiComposerThread): """Run Pungi with configs that produce module repositories.""" ctype = ContentType.module pungi_template_config_key = 'pungi.conf.module' - def copy_additional_pungi_files(self, pungi_conf_dir, template_env): + def _copy_additional_pungi_files(self, pungi_conf_dir, template_env): """ Generate and write the variants.xml file for this Pungi run. diff --git a/bodhi/tests/server/consumers/test_masher.py b/bodhi/tests/server/consumers/test_masher.py index 804670941b..dd1f6ca702 100644 --- a/bodhi/tests/server/consumers/test_masher.py +++ b/bodhi/tests/server/consumers/test_masher.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright © 2007-2017 Red Hat, Inc. +# Copyright © 2007-2018 Red Hat, Inc. # # This file is part of Bodhi. # @@ -35,8 +35,8 @@ from bodhi.server import buildsys, exceptions, log, push from bodhi.server.config import config -from bodhi.server.consumers.masher import ( - checkpoint, Masher, MasherThread, RPMMasherThread, ModuleMasherThread) +from bodhi.server.consumers.masher import (checkpoint, Masher, ComposerThread, RPMComposerThread, + ModuleComposerThread, PungiComposerThread) from bodhi.server.exceptions import LockedUpdateException from bodhi.server.models import ( Build, BuildrootOverride, Compose, ComposeState, Release, ReleaseState, RpmBuild, @@ -144,15 +144,15 @@ def setUp(self): super(TestMasher, self).setUp() self._new_mash_stage_dir = tempfile.mkdtemp() - # Since the MasherThread is a subclass of Thread and since it is already constructed before - # we have a chance to alter it, we need to change its superclass to be + # Since the ComposerThread is a subclass of Thread and since it is already constructed + # before we have a chance to alter it, we need to change its superclass to be # dummy_threading.Thread so that the test suite doesn't launch real Threads. Threads cannot # use the same database sessions, and that means that changes that threads make will not # appear in other thread's sessions, which cause a lot of problems in these tests. # Mock was not able to make this change since the __bases__ attribute cannot be removed, but # we don't really need this to be cleaned up since we don't want any tests launching theads # anyway. - MasherThread.__bases__ = (dummy_threading.Thread,) + ComposerThread.__bases__ = (dummy_threading.Thread,) test_config = base.original_config.copy() test_config['mash_stage_dir'] = self._new_mash_stage_dir test_config['mash_dir'] = os.path.join(self._new_mash_stage_dir, 'mash') @@ -192,8 +192,8 @@ def _generate_fake_pungi(self, masher_thread, tag, release): Return a function that is suitable for mock to replace the call to Popen that run Pungi. Args: - masher_thread (bodhi.server.consumers.masher.MasherThread): The MasherThread that Pungi - is running inside. + masher_thread (bodhi.server.consumers.masher.ComposerThread): The ComposerThread that + Pungi is running inside. tag (basestring): The type of tag you wish to mash ("stable_tag" or "testing_tag"). release (bodhi.server.models.Release): The Release you are mashing. Returns: @@ -212,7 +212,7 @@ def fake_pungi(*args, **kwargs): xmlns:rpm="http://linux.duke.edu/metadata/rpm"> 1508375628''' - # We need to fake Pungi having run or wait_for_mash() will fail to find the output dir + # We need to fake Pungi having run or _wait_for_pungi() will fail to find the output dir reqtype = 'updates' if tag == 'stable_tag' else 'updates-testing' mash_dir = os.path.join( masher_thread.mash_dir, @@ -348,12 +348,12 @@ def test_push_invalid_compose(self, publish): self.assertEqual(str(exc.exception), 'No row was found for one()') @mock.patch(**mock_taskotron_results) - @mock.patch('bodhi.server.consumers.masher.MasherThread.wait_for_mash') - @mock.patch('bodhi.server.consumers.masher.MasherThread.sanity_check_repo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.stage_repo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.generate_updateinfo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.wait_for_sync') - @mock.patch.object(MasherThread, 'determine_and_perform_tag_actions', mock_exc) + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._wait_for_pungi') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._sanity_check_repo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._stage_repo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._generate_updateinfo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._wait_for_sync') + @mock.patch.object(ComposerThread, 'determine_and_perform_tag_actions', mock_exc) @mock.patch('bodhi.server.notifications.publish') def test_update_locking(self, publish, *args): with self.db_factory() as session: @@ -387,11 +387,11 @@ def test_update_locking(self, publish, *args): pass @mock.patch(**mock_taskotron_results) - @mock.patch('bodhi.server.consumers.masher.MasherThread.wait_for_mash') - @mock.patch('bodhi.server.consumers.masher.MasherThread.sanity_check_repo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.stage_repo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.generate_updateinfo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.wait_for_sync') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._wait_for_pungi') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._sanity_check_repo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._stage_repo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._generate_updateinfo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._wait_for_sync') @mock.patch('bodhi.server.notifications.publish') def test_tags(self, publish, *args): # Make the build a buildroot override as well @@ -455,11 +455,11 @@ def test_tags(self, publish, *args): self.assertIsNone(up.request) @mock.patch(**mock_taskotron_results) - @mock.patch('bodhi.server.consumers.masher.MasherThread.wait_for_mash') - @mock.patch('bodhi.server.consumers.masher.MasherThread.sanity_check_repo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.stage_repo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.generate_updateinfo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.wait_for_sync') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._wait_for_pungi') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._sanity_check_repo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._stage_repo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._generate_updateinfo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._wait_for_sync') @mock.patch('bodhi.server.notifications.publish') def test_tag_ordering(self, publish, *args): """ @@ -504,16 +504,16 @@ def test_tag_ordering(self, publish, *args): (u'f17-updates-candidate', u'f17-updates-testing', u'bodhi-2.0-2.fc17')) @mock.patch(**mock_taskotron_results) - @mock.patch('bodhi.server.consumers.masher.MasherThread.wait_for_mash') - @mock.patch('bodhi.server.consumers.masher.MasherThread.sanity_check_repo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.stage_repo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.generate_updateinfo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.wait_for_sync') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._wait_for_pungi') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._sanity_check_repo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._stage_repo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._generate_updateinfo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._wait_for_sync') @mock.patch('bodhi.server.notifications.publish') @mock.patch('bodhi.server.mail._send_mail') def test_testing_digest(self, mail, *args): - t = RPMMasherThread(self._make_msg()['body']['msg']['composes'][0], - 'ralph', log, self.db_factory, self.tempdir) + t = RPMComposerThread(self._make_msg()['body']['msg']['composes'][0], + 'ralph', log, self.db_factory, self.tempdir) t.run() @@ -554,11 +554,11 @@ def test_testing_digest(self, mail, *args): '------------------------------------------------------------\n\n') % ( config.get('fedora_test_announce_list'), time.strftime('%Y'))), repr(body) - @mock.patch('bodhi.server.consumers.masher.MasherThread.save_state') + @mock.patch('bodhi.server.consumers.masher.ComposerThread.save_state') def test_mash_no_found_dirs(self, save_state): msg = self._make_msg() - t = RPMMasherThread(msg['body']['msg']['composes'][0], - 'ralph', log, self.db_factory, self.tempdir) + t = RPMComposerThread(msg['body']['msg']['composes'][0], + 'ralph', log, self.db_factory, self.tempdir) t.devnull = mock.MagicMock() t.id = 'f17-updates-testing' with self.db_factory() as session: @@ -571,7 +571,7 @@ def test_mash_no_found_dirs(self, save_state): fake_popen.poll.return_value = None fake_popen.returncode = 0 t._startyear = datetime.datetime.utcnow().year - t.wait_for_mash(fake_popen) + t._wait_for_pungi(fake_popen) assert False, "Mash without generated dirs did not crash" except Exception as ex: expected_error = ('We were unable to find a path with prefix ' @@ -580,11 +580,11 @@ def test_mash_no_found_dirs(self, save_state): assert str(ex) == expected_error t.db = None - @mock.patch('bodhi.server.consumers.masher.MasherThread.save_state') + @mock.patch('bodhi.server.consumers.masher.ComposerThread.save_state') def test_sanity_check_no_arches(self, save_state): msg = self._make_msg() - t = RPMMasherThread(msg['body']['msg']['composes'][0], - 'ralph', log, self.db_factory, self.tempdir) + t = RPMComposerThread(msg['body']['msg']['composes'][0], + 'ralph', log, self.db_factory, self.tempdir) t.devnull = mock.MagicMock() t.id = 'f17-updates-testing' with self.db_factory() as session: @@ -592,21 +592,21 @@ def test_sanity_check_no_arches(self, save_state): t.compose = session.query(Compose).one() t._checkpoints = {} t._startyear = datetime.datetime.utcnow().year - t.wait_for_mash(self._generate_fake_pungi(t, 'testing_tag', t.compose.release)()) + t._wait_for_pungi(self._generate_fake_pungi(t, 'testing_tag', t.compose.release)()) t.db = None # test without any arches try: - t.sanity_check_repo() + t._sanity_check_repo() assert False, "Sanity check didn't fail with empty dir" except Exception: pass - @mock.patch('bodhi.server.consumers.masher.MasherThread.save_state') + @mock.patch('bodhi.server.consumers.masher.ComposerThread.save_state') def test_sanity_check_valid(self, save_state): msg = self._make_msg() - t = RPMMasherThread(msg['body']['msg']['composes'][0], - 'ralph', log, self.db_factory, self.tempdir) + t = RPMComposerThread(msg['body']['msg']['composes'][0], + 'ralph', log, self.db_factory, self.tempdir) t.devnull = mock.MagicMock() t.id = 'f17-updates-testing' with self.db_factory() as session: @@ -614,7 +614,7 @@ def test_sanity_check_valid(self, save_state): t.compose = session.query(Compose).one() t._checkpoints = {} t._startyear = datetime.datetime.utcnow().year - t.wait_for_mash(self._generate_fake_pungi(t, 'testing_tag', t.compose.release)()) + t._wait_for_pungi(self._generate_fake_pungi(t, 'testing_tag', t.compose.release)()) t.db = None # test with valid repodata @@ -635,13 +635,13 @@ def test_sanity_check_valid(self, save_state): 'test.src.rpm'), 'w') as tf: tf.write('bar') - t.sanity_check_repo() + t._sanity_check_repo() - @mock.patch('bodhi.server.consumers.masher.MasherThread.save_state') + @mock.patch('bodhi.server.consumers.masher.ComposerThread.save_state') def test_sanity_check_broken_repodata(self, save_state): msg = self._make_msg() - t = RPMMasherThread(msg['body']['msg']['composes'][0], - 'ralph', log, self.db_factory, self.tempdir) + t = RPMComposerThread(msg['body']['msg']['composes'][0], + 'ralph', log, self.db_factory, self.tempdir) t.devnull = mock.MagicMock() t.id = 'f17-updates-testing' with self.db_factory() as session: @@ -649,7 +649,7 @@ def test_sanity_check_broken_repodata(self, save_state): t.compose = session.query(Compose).one() t._startyear = datetime.datetime.utcnow().year t._checkpoints = {} - t.wait_for_mash(self._generate_fake_pungi(t, 'testing_tag', t.compose.release)()) + t._wait_for_pungi(self._generate_fake_pungi(t, 'testing_tag', t.compose.release)()) t.db = None # test with valid repodata @@ -666,18 +666,18 @@ def test_sanity_check_broken_repodata(self, save_state): f.write(repomd[:-10]) try: - t.sanity_check_repo() + t._sanity_check_repo() assert False, 'Busted metadata passed' except exceptions.RepodataException: pass save_state.assert_called_once_with(ComposeState.punging) - @mock.patch('bodhi.server.consumers.masher.MasherThread.save_state') + @mock.patch('bodhi.server.consumers.masher.ComposerThread.save_state') def test_sanity_check_symlink(self, save_state): msg = self._make_msg() - t = RPMMasherThread(msg['body']['msg']['composes'][0], - 'ralph', log, self.db_factory, self.tempdir) + t = RPMComposerThread(msg['body']['msg']['composes'][0], + 'ralph', log, self.db_factory, self.tempdir) t.devnull = mock.MagicMock() t.id = 'f17-updates-testing' with self.db_factory() as session: @@ -685,7 +685,7 @@ def test_sanity_check_symlink(self, save_state): t.compose = session.query(Compose).one() t._checkpoints = {} t._startyear = datetime.datetime.utcnow().year - t.wait_for_mash(self._generate_fake_pungi(t, 'testing_tag', t.compose.release)()) + t._wait_for_pungi(self._generate_fake_pungi(t, 'testing_tag', t.compose.release)()) t.db = None # test with valid repodata @@ -702,16 +702,16 @@ def test_sanity_check_symlink(self, save_state): 'Packages', 'a', 'test.src.rpm')) try: - t.sanity_check_repo() + t._sanity_check_repo() assert False, "Symlinks passed" except Exception as ex: assert str(ex) == "Symlinks found" - @mock.patch('bodhi.server.consumers.masher.MasherThread.save_state') + @mock.patch('bodhi.server.consumers.masher.ComposerThread.save_state') def test_sanity_check_directories_missing(self, save_state): msg = self._make_msg() - t = RPMMasherThread(msg['body']['msg']['composes'][0], - 'ralph', log, self.db_factory, self.tempdir) + t = RPMComposerThread(msg['body']['msg']['composes'][0], + 'ralph', log, self.db_factory, self.tempdir) t.devnull = mock.MagicMock() t.id = 'f17-updates-testing' with self.db_factory() as session: @@ -719,7 +719,7 @@ def test_sanity_check_directories_missing(self, save_state): t.compose = session.query(Compose).one() t._checkpoints = {} t._startyear = datetime.datetime.utcnow().year - t.wait_for_mash(self._generate_fake_pungi(t, 'testing_tag', t.compose.release)()) + t._wait_for_pungi(self._generate_fake_pungi(t, 'testing_tag', t.compose.release)()) t.db = None # test with valid repodata @@ -731,29 +731,17 @@ def test_sanity_check_directories_missing(self, save_state): mkmetadatadir(os.path.join(t.path, 'compose', 'Everything', 'source', 'tree')) try: - t.sanity_check_repo() + t._sanity_check_repo() assert False, "Missing directories passed" except OSError as oex: assert oex.errno == errno.ENOENT - def test_stage(self): - t = MasherThread(self._make_msg()['body']['msg']['composes'][0], - 'ralph', log, self.db_factory, self.tempdir) - t.id = 'f17-updates-testing' - t.path = os.path.join(self.tempdir, 'latest-f17-updates-testing') - os.makedirs(t.path) - t.stage_repo() - stage_dir = config.get('mash_stage_dir') - link = os.path.join(stage_dir, t.id) - self.assertTrue(os.path.islink(link)) - assert os.readlink(link) == t.path - @mock.patch(**mock_taskotron_results) - @mock.patch('bodhi.server.consumers.masher.MasherThread.wait_for_mash') - @mock.patch('bodhi.server.consumers.masher.MasherThread.sanity_check_repo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.stage_repo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.generate_updateinfo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.wait_for_sync') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._wait_for_pungi') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._sanity_check_repo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._stage_repo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._generate_updateinfo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._wait_for_sync') @mock.patch('bodhi.server.notifications.publish') def test_security_update_priority(self, publish, *args): with self.db_factory() as db: @@ -833,11 +821,11 @@ def test_security_update_priority(self, publish, *args): topic='mashtask.complete')) @mock.patch(**mock_taskotron_results) - @mock.patch('bodhi.server.consumers.masher.MasherThread.wait_for_mash') - @mock.patch('bodhi.server.consumers.masher.MasherThread.sanity_check_repo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.stage_repo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.generate_updateinfo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.wait_for_sync') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._wait_for_pungi') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._sanity_check_repo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._stage_repo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._generate_updateinfo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._wait_for_sync') @mock.patch('bodhi.server.notifications.publish') def test_security_update_priority_testing(self, publish, *args): with self.db_factory() as db: @@ -910,11 +898,11 @@ def test_security_update_priority_testing(self, publish, *args): topic='mashtask.complete')) @mock.patch(**mock_taskotron_results) - @mock.patch('bodhi.server.consumers.masher.MasherThread.wait_for_mash') - @mock.patch('bodhi.server.consumers.masher.MasherThread.sanity_check_repo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.stage_repo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.generate_updateinfo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.wait_for_sync') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._wait_for_pungi') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._sanity_check_repo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._stage_repo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._generate_updateinfo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._wait_for_sync') @mock.patch('bodhi.server.notifications.publish') def test_security_updates_parallel(self, publish, *args): with self.db_factory() as db: @@ -1000,17 +988,17 @@ def test_mash_invalid_ctype(self, publish, *args): def test_base_masher_pungi_not_implemented(self, *args): msg = self._make_msg() - t = MasherThread(msg['body']['msg']['composes'][0], 'ralph', log, self.db_factory, - self.tempdir) + t = PungiComposerThread(msg['body']['msg']['composes'][0], 'ralph', log, self.db_factory, + self.tempdir) with self.assertRaises(NotImplementedError): - t.copy_additional_pungi_files(None, None) + t._copy_additional_pungi_files(None, None) @mock.patch(**mock_taskotron_results) - @mock.patch('bodhi.server.consumers.masher.MasherThread.sanity_check_repo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.stage_repo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.generate_updateinfo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.wait_for_sync') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._sanity_check_repo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._stage_repo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._generate_updateinfo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._wait_for_sync') @mock.patch('bodhi.server.notifications.publish') @mock.patch.dict( config, @@ -1019,8 +1007,8 @@ def test_mash_early_exit(self, publish, *args): # Set the request to stable right out the gate so we can test gating self.set_stable_request(u'bodhi-2.0-1.fc17') msg = self._make_msg() - t = RPMMasherThread(msg['body']['msg']['composes'][0], - 'ralph', log, self.db_factory, self.tempdir) + t = RPMComposerThread(msg['body']['msg']['composes'][0], + 'ralph', log, self.db_factory, self.tempdir) t.run() @@ -1032,17 +1020,17 @@ def test_mash_early_exit(self, publish, *args): self.assertEqual(t._checkpoints, {'determine_and_perform_tag_actions': True}) @mock.patch(**mock_taskotron_results) - @mock.patch('bodhi.server.consumers.masher.MasherThread.sanity_check_repo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.stage_repo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.generate_updateinfo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.wait_for_sync') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._sanity_check_repo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._stage_repo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._generate_updateinfo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._wait_for_sync') @mock.patch('bodhi.server.notifications.publish') def test_mash_late_exit(self, publish, *args): # Set the request to stable right out the gate so we can test gating self.set_stable_request(u'bodhi-2.0-1.fc17') msg = self._make_msg() - t = RPMMasherThread(msg['body']['msg']['composes'][0], - 'ralph', log, self.db_factory, self.tempdir) + t = RPMComposerThread(msg['body']['msg']['composes'][0], + 'ralph', log, self.db_factory, self.tempdir) with self.db_factory() as session: with tempfile.NamedTemporaryFile(delete=False) as script: @@ -1060,16 +1048,17 @@ def test_mash_late_exit(self, publish, *args): self.assertEqual(t._checkpoints, {'determine_and_perform_tag_actions': True}) @mock.patch(**mock_taskotron_results) - @mock.patch('bodhi.server.consumers.masher.MasherThread.sanity_check_repo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.stage_repo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.wait_for_sync') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._sanity_check_repo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._stage_repo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._wait_for_sync') @mock.patch('bodhi.server.notifications.publish') def test_mash(self, publish, *args): # Set the request to stable right out the gate so we can test gating self.set_stable_request(u'bodhi-2.0-1.fc17') msg = self._make_msg() - t = RPMMasherThread(msg['body']['msg']['composes'][0], - 'ralph', log, self.db_factory, self.tempdir) + mash_dir = os.path.join(self.tempdir, 'cool_dir') + t = RPMComposerThread(msg['body']['msg']['composes'][0], + 'ralph', log, self.db_factory, mash_dir) with self.db_factory() as session: with mock.patch('bodhi.server.consumers.masher.subprocess.Popen') as Popen: @@ -1101,18 +1090,19 @@ def test_mash(self, publish, *args): self.assertEqual( t._checkpoints, {'completed_repo': os.path.join( - self.tempdir, 'Fedora-17-updates-{}{:02}{:02}.0'.format(d.year, d.month, d.day)), + mash_dir, 'Fedora-17-updates-{}{:02}{:02}.0'.format(d.year, d.month, d.day)), 'determine_and_perform_tag_actions': True, 'modify_bugs': True, 'send_stable_announcements': True, 'send_testing_digest': True, 'status_comments': True}) + self.assertTrue(os.path.exists(mash_dir)) @mock.patch(**mock_taskotron_results) - @mock.patch('bodhi.server.consumers.masher.MasherThread.sanity_check_repo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.stage_repo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.generate_updateinfo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.wait_for_sync') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._sanity_check_repo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._stage_repo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._generate_updateinfo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._wait_for_sync') @mock.patch('bodhi.server.notifications.publish') def test_mash_module(self, publish, *args): with self.db_factory() as db: @@ -1144,8 +1134,8 @@ def test_mash_module(self, publish, *args): Release._tag_cache = None msg = self._make_msg(['--releases', 'F27M']) - t = ModuleMasherThread(msg['body']['msg']['composes'][0], - 'puiterwijk', log, self.db_factory, self.tempdir) + t = ModuleComposerThread(msg['body']['msg']['composes'][0], + 'puiterwijk', log, self.db_factory, self.tempdir) with self.db_factory() as session: with mock.patch('bodhi.server.consumers.masher.subprocess.Popen') as Popen: @@ -1186,17 +1176,17 @@ def test_mash_module(self, publish, *args): 'status_comments': True}) @mock.patch(**mock_failed_taskotron_results) - @mock.patch('bodhi.server.consumers.masher.MasherThread.sanity_check_repo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.stage_repo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.generate_updateinfo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.wait_for_sync') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._sanity_check_repo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._stage_repo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._generate_updateinfo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._wait_for_sync') @mock.patch('bodhi.server.notifications.publish') def test_failed_gating(self, publish, *args): # Set the request to stable right out the gate so we can test gating self.set_stable_request(u'bodhi-2.0-1.fc17') msg = self._make_msg() - t = RPMMasherThread( + t = RPMComposerThread( msg['body']['msg']['composes'][0], 'ralph', log, self.db_factory, self.tempdir) with self.db_factory() as session: @@ -1235,18 +1225,18 @@ def test_failed_gating(self, publish, *args): 'status_comments': True}) @mock.patch(**mock_absent_taskotron_results) - @mock.patch('bodhi.server.consumers.masher.MasherThread.sanity_check_repo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.stage_repo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.generate_updateinfo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.wait_for_sync') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._sanity_check_repo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._stage_repo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._generate_updateinfo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._wait_for_sync') @mock.patch('bodhi.server.notifications.publish') def test_absent_gating(self, publish, *args): # Set the request to stable right out the gate so we can test gating self.set_stable_request(u'bodhi-2.0-1.fc17') msg = self._make_msg() - t = RPMMasherThread(msg['body']['msg']['composes'][0], 'ralph', log, self.db_factory, - self.tempdir) + t = RPMComposerThread(msg['body']['msg']['composes'][0], 'ralph', log, self.db_factory, + self.tempdir) with self.db_factory() as session: with mock.patch('bodhi.server.consumers.masher.subprocess.Popen') as Popen: @@ -1283,11 +1273,11 @@ def test_absent_gating(self, publish, *args): 'send_testing_digest': True, 'status_comments': True}) - @mock.patch('bodhi.server.consumers.masher.MasherThread.wait_for_mash') - @mock.patch('bodhi.server.consumers.masher.MasherThread.sanity_check_repo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.stage_repo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.generate_updateinfo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.wait_for_sync') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._wait_for_pungi') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._sanity_check_repo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._stage_repo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._generate_updateinfo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._wait_for_sync') @mock.patch('bodhi.server.notifications.publish') @mock.patch('bodhi.server.util.cmd') @mock.patch('bodhi.server.bugs.bugtracker.modified') @@ -1307,19 +1297,19 @@ def test_modify_testing_bugs(self, on_qa, modified, *args): on_qa.assert_called_once_with(12345, expected_message) @mock.patch(**mock_taskotron_results) - @mock.patch('bodhi.server.consumers.masher.MasherThread.wait_for_mash') - @mock.patch('bodhi.server.consumers.masher.MasherThread.sanity_check_repo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.stage_repo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.generate_updateinfo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.wait_for_sync') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._wait_for_pungi') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._sanity_check_repo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._stage_repo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._generate_updateinfo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._wait_for_sync') @mock.patch('bodhi.server.notifications.publish') @mock.patch('bodhi.server.bugs.bugtracker.comment') @mock.patch('bodhi.server.bugs.bugtracker.close') def test_modify_stable_bugs(self, close, comment, *args): self.set_stable_request(u'bodhi-2.0-1.fc17') msg = self._make_msg() - t = RPMMasherThread(msg['body']['msg']['composes'][0], - 'ralph', log, self.db_factory, self.tempdir) + t = RPMComposerThread(msg['body']['msg']['composes'][0], + 'ralph', log, self.db_factory, self.tempdir) t.run() @@ -1330,11 +1320,11 @@ def test_modify_stable_bugs(self, close, comment, *args): u'problems still persist, please make note of it in this bug report.')) @mock.patch(**mock_taskotron_results) - @mock.patch('bodhi.server.consumers.masher.MasherThread.wait_for_mash') - @mock.patch('bodhi.server.consumers.masher.MasherThread.sanity_check_repo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.stage_repo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.generate_updateinfo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.wait_for_sync') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._wait_for_pungi') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._sanity_check_repo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._stage_repo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._generate_updateinfo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._wait_for_sync') @mock.patch('bodhi.server.notifications.publish') @mock.patch('bodhi.server.util.cmd') def test_status_comment_testing(self, *args): @@ -1350,11 +1340,11 @@ def test_status_comment_testing(self, *args): self.assertEquals(up.comments[-1]['text'], u'This update has been pushed to testing.') @mock.patch(**mock_taskotron_results) - @mock.patch('bodhi.server.consumers.masher.MasherThread.wait_for_mash') - @mock.patch('bodhi.server.consumers.masher.MasherThread.sanity_check_repo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.stage_repo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.generate_updateinfo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.wait_for_sync') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._wait_for_pungi') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._sanity_check_repo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._stage_repo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._generate_updateinfo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._wait_for_sync') @mock.patch('bodhi.server.notifications.publish') @mock.patch('bodhi.server.util.cmd') def test_status_comment_stable(self, *args): @@ -1371,16 +1361,16 @@ def test_status_comment_stable(self, *args): self.assertEquals(up.comments[-1]['text'], u'This update has been pushed to stable.') @mock.patch(**mock_taskotron_results) - @mock.patch('bodhi.server.consumers.masher.MasherThread.wait_for_mash') - @mock.patch('bodhi.server.consumers.masher.MasherThread.sanity_check_repo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.stage_repo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.generate_updateinfo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.wait_for_sync') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._wait_for_pungi') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._sanity_check_repo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._stage_repo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._generate_updateinfo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._wait_for_sync') @mock.patch('bodhi.server.notifications.publish') def test_get_security_updates(self, *args): msg = self._make_msg() - t = MasherThread(msg['body']['msg']['composes'][0], - 'ralph', log, self.db_factory, self.tempdir) + t = ComposerThread(msg['body']['msg']['composes'][0], + 'ralph', log, self.db_factory, self.tempdir) with self.db_factory() as session: t.db = session u = session.query(Update).one() @@ -1396,11 +1386,11 @@ def test_get_security_updates(self, *args): self.assertEquals(updates[0].title, u.builds[0].nvr) @mock.patch(**mock_taskotron_results) - @mock.patch('bodhi.server.consumers.masher.MasherThread.wait_for_mash') - @mock.patch('bodhi.server.consumers.masher.MasherThread.sanity_check_repo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.stage_repo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.generate_updateinfo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.wait_for_sync') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._wait_for_pungi') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._sanity_check_repo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._stage_repo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._generate_updateinfo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._wait_for_sync') @mock.patch('bodhi.server.notifications.publish') @mock.patch('bodhi.server.util.cmd') def test_unlock_updates(self, *args): @@ -1417,15 +1407,15 @@ def test_unlock_updates(self, *args): self.assertEquals(up.status, UpdateStatus.stable) @mock.patch(**mock_taskotron_results) - @mock.patch('bodhi.server.consumers.masher.MasherThread.wait_for_mash') - @mock.patch('bodhi.server.consumers.masher.MasherThread.sanity_check_repo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.stage_repo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.generate_updateinfo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.wait_for_sync') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._wait_for_pungi') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._sanity_check_repo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._stage_repo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._generate_updateinfo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._wait_for_sync') @mock.patch('bodhi.server.notifications.publish') @mock.patch('bodhi.server.util.cmd') def test_resume_push(self, *args): - with mock.patch.object(MasherThread, 'generate_testing_digest', mock_exc): + with mock.patch.object(ComposerThread, 'generate_testing_digest', mock_exc): with self.db_factory() as session: up = session.query(Update).one() up.request = UpdateRequest.testing @@ -1451,11 +1441,11 @@ def test_resume_push(self, *args): self.assertEquals(up.request, None) @mock.patch(**mock_taskotron_results) - @mock.patch('bodhi.server.consumers.masher.MasherThread.wait_for_mash') - @mock.patch('bodhi.server.consumers.masher.MasherThread.sanity_check_repo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.stage_repo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.generate_updateinfo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.wait_for_sync') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._wait_for_pungi') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._sanity_check_repo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._stage_repo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._generate_updateinfo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._wait_for_sync') @mock.patch('bodhi.server.notifications.publish') @mock.patch('bodhi.server.util.cmd') def test_stable_requirements_met_during_push(self, *args): @@ -1464,7 +1454,7 @@ def test_stable_requirements_met_during_push(self, *args): pushed to testing """ # Simulate a failed push - with mock.patch.object(MasherThread, 'determine_and_perform_tag_actions', mock_exc): + with mock.patch.object(ComposerThread, 'determine_and_perform_tag_actions', mock_exc): with self.db_factory() as session: up = session.query(Update).one() up.request = UpdateRequest.testing @@ -1507,11 +1497,11 @@ def test_stable_requirements_met_during_push(self, *args): self.assertEquals(up.request, UpdateRequest.batched) @mock.patch(**mock_taskotron_results) - @mock.patch('bodhi.server.consumers.masher.MasherThread.wait_for_mash') - @mock.patch('bodhi.server.consumers.masher.MasherThread.sanity_check_repo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.stage_repo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.generate_updateinfo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.wait_for_sync') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._wait_for_pungi') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._sanity_check_repo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._stage_repo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._generate_updateinfo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._wait_for_sync') @mock.patch('bodhi.server.notifications.publish') def test_push_timestamps(self, publish, *args): with self.db_factory() as session: @@ -1552,11 +1542,11 @@ def test_push_timestamps(self, publish, *args): self.assertIsNotNone(up.date_stable) @mock.patch(**mock_taskotron_results) - @mock.patch('bodhi.server.consumers.masher.MasherThread.wait_for_mash') - @mock.patch('bodhi.server.consumers.masher.MasherThread.sanity_check_repo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.stage_repo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.generate_updateinfo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.wait_for_sync') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._wait_for_pungi') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._sanity_check_repo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._stage_repo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._generate_updateinfo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._wait_for_sync') @mock.patch('bodhi.server.notifications.publish') def test_obsolete_older_updates(self, publish, *args): otherbuild = u'bodhi-2.0-2.fc17' @@ -1593,11 +1583,11 @@ def test_obsolete_older_updates(self, publish, *args): self.assertEquals(up.request, None) @mock.patch(**mock_taskotron_results) - @mock.patch('bodhi.server.consumers.masher.MasherThread.wait_for_mash') - @mock.patch('bodhi.server.consumers.masher.MasherThread.sanity_check_repo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.stage_repo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.generate_updateinfo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.wait_for_sync') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._wait_for_pungi') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._sanity_check_repo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._stage_repo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._generate_updateinfo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._wait_for_sync') @mock.patch('bodhi.server.notifications.publish') @mock.patch('bodhi.server.consumers.masher.log.exception') @mock.patch('bodhi.server.models.BuildrootOverride.expire', side_effect=Exception()) @@ -1680,11 +1670,11 @@ def _ensure_three_mashes_in_same_batch(self): Release._tag_cache = None @mock.patch(**mock_taskotron_results) - @mock.patch('bodhi.server.consumers.masher.MasherThread.wait_for_mash') - @mock.patch('bodhi.server.consumers.masher.MasherThread.sanity_check_repo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.stage_repo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.generate_updateinfo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.wait_for_sync') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._wait_for_pungi') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._sanity_check_repo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._stage_repo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._generate_updateinfo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._wait_for_sync') @mock.patch('bodhi.server.notifications.publish') def test_work_max_concurrent_mashes_1(self, publish, *args): """Assert that we don't launch more than 1 max_concurrent_mashes.""" @@ -1702,11 +1692,11 @@ def test_work_max_concurrent_mashes_1(self, publish, *args): ('Waiting on %d mashes for priority %s', 1, 0)]) @mock.patch(**mock_taskotron_results) - @mock.patch('bodhi.server.consumers.masher.MasherThread.wait_for_mash') - @mock.patch('bodhi.server.consumers.masher.MasherThread.sanity_check_repo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.stage_repo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.generate_updateinfo') - @mock.patch('bodhi.server.consumers.masher.MasherThread.wait_for_sync') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._wait_for_pungi') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._sanity_check_repo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._stage_repo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._generate_updateinfo') + @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._wait_for_sync') @mock.patch('bodhi.server.notifications.publish') def test_work_max_concurrent_mashes_2(self, publish, *args): """Assert that we don't launch more than 2 max_concurrent_mashes.""" @@ -1723,16 +1713,16 @@ def test_work_max_concurrent_mashes_2(self, publish, *args): self.assertEqual(waiting_messages, [('Waiting on %d mashes for priority %s', 2, 0)]) -class MasherThreadBaseTestCase(base.BaseTestCase): +class ComposerThreadBaseTestCase(base.BaseTestCase): """ This test class has common setUp() and tearDown() methods that are useful for testing the - MasherThread class. + ComposerThread class. """ def setUp(self): """ Set up the test conditions. """ - super(MasherThreadBaseTestCase, self).setUp() + super(ComposerThreadBaseTestCase, self).setUp() buildsys.setup_buildsystem({'buildsystem': 'dev'}) self.tempdir = tempfile.mkdtemp() @@ -1740,7 +1730,7 @@ def tearDown(self): """ Clean up after the tests. """ - super(MasherThreadBaseTestCase, self).tearDown() + super(ComposerThreadBaseTestCase, self).tearDown() shutil.rmtree(self.tempdir) buildsys.teardown_buildsystem() @@ -1754,8 +1744,8 @@ def _make_msg(self, extra_push_args=None): return _make_msg(base.TransactionalSessionMaker(self.Session), extra_push_args) -class TestMasherThread__get_master_repomd_url(MasherThreadBaseTestCase): - """This test class contains tests for the MasherThread._get_master_repomd_url() method.""" +class TestPungiComposerThread__get_master_repomd_url(ComposerThreadBaseTestCase): + """This class contains tests for the PungiComposerThread._get_master_repomd_url() method.""" @mock.patch.dict( 'bodhi.server.consumers.masher.config', {'fedora_17_primary_arches': 'armhfp x86_64', @@ -1769,8 +1759,8 @@ def test_alternative_arch(self): arches and the arch being looked up is not in the primary arch list. """ msg = self._make_msg() - t = MasherThread(msg['body']['msg']['composes'][0], - 'bowlofeggs', log, self.Session, self.tempdir) + t = PungiComposerThread(msg['body']['msg']['composes'][0], + 'bowlofeggs', log, self.Session, self.tempdir) t.compose = Compose.from_dict(self.db, msg['body']['msg']['composes'][0]) url = t._get_master_repomd_url('aarch64') @@ -1791,8 +1781,8 @@ def test_master_repomd_undefined(self): the release. """ msg = self._make_msg() - t = MasherThread(msg['body']['msg']['composes'][0], - 'bowlofeggs', log, self.Session, self.tempdir) + t = PungiComposerThread(msg['body']['msg']['composes'][0], + 'bowlofeggs', log, self.Session, self.tempdir) t.compose = Compose.from_dict(self.db, msg['body']['msg']['composes'][0]) with self.assertRaises(ValueError) as exc: @@ -1814,8 +1804,8 @@ def test_primary_arch(self): arches and the arch being looked up is primary. """ msg = self._make_msg() - t = MasherThread(msg['body']['msg']['composes'][0], - 'bowlofeggs', log, self.Session, self.tempdir) + t = PungiComposerThread(msg['body']['msg']['composes'][0], + 'bowlofeggs', log, self.Session, self.tempdir) t.compose = Compose.from_dict(self.db, msg['body']['msg']['composes'][0]) url = t._get_master_repomd_url('x86_64') @@ -1837,8 +1827,8 @@ def test_primary_arches_undefined(self): arches defined in the config file. """ msg = self._make_msg() - t = MasherThread(msg['body']['msg']['composes'][0], - 'bowlofeggs', log, self.Session, self.tempdir) + t = PungiComposerThread(msg['body']['msg']['composes'][0], + 'bowlofeggs', log, self.Session, self.tempdir) t.compose = Compose.from_dict(self.db, msg['body']['msg']['composes'][0]) url = t._get_master_repomd_url('aarch64') @@ -1849,8 +1839,8 @@ def test_primary_arches_undefined(self): ) -class TestMasherThread__perform_tag_actions(MasherThreadBaseTestCase): - """This test class contains tests for the MasherThread._perform_tag_actions() method.""" +class TestComposerThread__perform_tag_actions(ComposerThreadBaseTestCase): + """This test class contains tests for the ComposerThread._perform_tag_actions() method.""" @mock.patch('bodhi.server.consumers.masher.buildsys.wait_for_tasks') def test_with_failed_tasks(self, wait_for_tasks): """ @@ -1858,8 +1848,8 @@ def test_with_failed_tasks(self, wait_for_tasks): """ wait_for_tasks.return_value = ['failed_task_1'] msg = self._make_msg() - t = MasherThread(msg['body']['msg']['composes'][0], - 'bowlofeggs', log, self.Session, self.tempdir) + t = ComposerThread(msg['body']['msg']['composes'][0], + 'bowlofeggs', log, self.Session, self.tempdir) t.compose = Compose.from_dict(self.db, msg['body']['msg']['composes'][0]) t.move_tags_async.append( (u'f26-updates-candidate', u'f26-updates-testing', u'bodhi-2.3.2-1.fc26')) @@ -1874,15 +1864,15 @@ def test_with_failed_tasks(self, wait_for_tasks): [('f26-updates-candidate', 'f26-updates-testing', 'bodhi-2.3.2-1.fc26')]) -class TestMasherThread_check_all_karma_thresholds(MasherThreadBaseTestCase): - """Test the MasherThread.check_all_karma_thresholds() method.""" +class TestComposerThread_check_all_karma_thresholds(ComposerThreadBaseTestCase): + """Test the ComposerThread.check_all_karma_thresholds() method.""" @mock.patch('bodhi.server.models.Update.check_karma_thresholds', mock.MagicMock(side_effect=exceptions.BodhiException('BOOM'))) def test_BodhiException(self): """Assert that a raised BodhiException gets caught and logged.""" msg = self._make_msg() - t = MasherThread(msg['body']['msg']['composes'][0], - 'bowlofeggs', log, self.Session, self.tempdir) + t = ComposerThread(msg['body']['msg']['composes'][0], + 'bowlofeggs', log, self.Session, self.tempdir) t.compose = Compose.from_dict(self.db, msg['body']['msg']['composes'][0]) t.db = self.db t.log.exception = mock.MagicMock() @@ -1892,8 +1882,8 @@ def test_BodhiException(self): t.log.exception.assert_called_once_with('Problem checking karma thresholds') -class TestMasherThread_eject_from_mash(MasherThreadBaseTestCase): - """This test class contains tests for the MasherThread.eject_from_mash() method.""" +class TestComposerThread_eject_from_mash(ComposerThreadBaseTestCase): + """This test class contains tests for the ComposerThread.eject_from_mash() method.""" @mock.patch('bodhi.server.notifications.publish') def test_testing_request(self, publish): """ @@ -1903,8 +1893,8 @@ def test_testing_request(self, publish): up.request = UpdateRequest.testing self.db.commit() msg = self._make_msg() - t = MasherThread(msg['body']['msg']['composes'][0], - 'bowlofeggs', log, self.Session, self.tempdir) + t = ComposerThread(msg['body']['msg']['composes'][0], + 'bowlofeggs', log, self.Session, self.tempdir) # t.work() would normally set these up for us, so we'll just fake it t.compose = Compose.from_dict(self.db, msg['body']['msg']['composes'][0]) t.db = self.Session() @@ -1926,28 +1916,28 @@ def test_testing_request(self, publish): self.assertEqual(len(t.compose.updates), 0) -class TestMasherThread_init_state(MasherThreadBaseTestCase): - """This test class contains tests for the MasherThread.init_state() method.""" - def test_creates_mash_dir(self): - """Assert that mash_dir gets created if it doesn't exist.""" - mash_dir = os.path.join(self.tempdir, 'cool_dir') - msg = self._make_msg() - t = MasherThread(msg['body']['msg']['composes'][0], - 'bowlofeggs', log, self.Session, mash_dir) - # t.work() would normally set this up for us, so we'll just fake it - t.id = getattr(self.db.query(Release).one(), '{}_tag'.format('stable')) +class TestComposerThread_load_state(ComposerThreadBaseTestCase): + """This test class contains tests for the ComposerThread.load_state() method.""" + def test_with_completed_repo(self): + """Test when there is a completed_repo in the checkpoints.""" + t = ComposerThread(self._make_msg()['body']['msg']['composes'][0], + 'bowlofeggs', log, self.Session, self.tempdir) + t._checkpoints = {'cool': 'checkpoint'} + t.compose = self.db.query(Compose).one() + t.compose.checkpoints = json.dumps({'other': 'checkpoint', 'completed_repo': '/path/to/it'}) + t.db = self.db - t.init_state() + t.load_state() - self.assertTrue(os.path.exists(mash_dir)) + self.assertEqual(t._checkpoints, {'other': 'checkpoint', 'completed_repo': '/path/to/it'}) -class TestMasherThread_load_state(MasherThreadBaseTestCase): - """This test class contains tests for the MasherThread.load_state() method.""" +class TestPungiComposerThread_load_state(ComposerThreadBaseTestCase): + """This test class contains tests for the PungiComposerThread.load_state() method.""" def test_with_completed_repo(self): """Test when there is a completed_repo in the checkpoints.""" - t = MasherThread(self._make_msg()['body']['msg']['composes'][0], - 'bowlofeggs', log, self.Session, self.tempdir) + t = PungiComposerThread(self._make_msg()['body']['msg']['composes'][0], + 'bowlofeggs', log, self.Session, self.tempdir) t._checkpoints = {'cool': 'checkpoint'} t.compose = self.db.query(Compose).one() t.compose.checkpoints = json.dumps({'other': 'checkpoint', 'completed_repo': '/path/to/it'}) @@ -1960,8 +1950,8 @@ def test_with_completed_repo(self): def test_without_completed_repo(self): """Test when there is not a completed_repo in the checkpoints.""" - t = MasherThread(self._make_msg()['body']['msg']['composes'][0], - 'bowlofeggs', log, self.Session, self.tempdir) + t = PungiComposerThread(self._make_msg()['body']['msg']['composes'][0], + 'bowlofeggs', log, self.Session, self.tempdir) t._checkpoints = {'cool': 'checkpoint'} t.compose = self.db.query(Compose).one() t.compose.checkpoints = json.dumps({'other': 'checkpoint'}) @@ -1973,12 +1963,12 @@ def test_without_completed_repo(self): self.assertEqual(t.path, None) -class TestMasherThread_remove_state(MasherThreadBaseTestCase): +class TestComposerThread_remove_state(ComposerThreadBaseTestCase): """Test the remove_state() method.""" def test_remove_state(self): """Assert that remove_state() deletes the Compose.""" - t = MasherThread(self._make_msg()['body']['msg']['composes'][0], - 'bowlofeggs', log, self.Session, self.tempdir) + t = ComposerThread(self._make_msg()['body']['msg']['composes'][0], + 'bowlofeggs', log, self.Session, self.tempdir) t.compose = self.db.query(Compose).one() t.db = self.db @@ -1988,12 +1978,12 @@ def test_remove_state(self): self.assertEqual(self.db.query(Compose).count(), 0) -class TestMasherThread_save_state(MasherThreadBaseTestCase): - """This test class contains tests for the MasherThread.save_state() method.""" +class TestComposerThread_save_state(ComposerThreadBaseTestCase): + """This test class contains tests for the ComposerThread.save_state() method.""" def test_with_state(self): """Test the optional state parameter.""" - t = MasherThread(self._make_msg()['body']['msg']['composes'][0], - 'bowlofeggs', log, self.Session, self.tempdir) + t = ComposerThread(self._make_msg()['body']['msg']['composes'][0], + 'bowlofeggs', log, self.Session, self.tempdir) t._checkpoints = {'cool': 'checkpoint'} t.compose = self.db.query(Compose).one() t.db = self.db @@ -2008,8 +1998,8 @@ def test_with_state(self): def test_without_state(self): """Test without the optional state parameter.""" - t = MasherThread(self._make_msg()['body']['msg']['composes'][0], - 'bowlofeggs', log, self.Session, self.tempdir) + t = ComposerThread(self._make_msg()['body']['msg']['composes'][0], + 'bowlofeggs', log, self.Session, self.tempdir) t._checkpoints = {'cool': 'checkpoint'} t.compose = self.db.query(Compose).one() t.db = self.db @@ -2023,8 +2013,8 @@ def test_without_state(self): t.db.commit.assert_called_once_with() -class TestMasherThread_wait_for_sync(MasherThreadBaseTestCase): - """This test class contains tests for the MasherThread.wait_for_sync() method.""" +class TestPungiComposerThread__wait_for_sync(ComposerThreadBaseTestCase): + """This test class contains tests for the PungiComposerThread._wait_for_sync() method.""" @mock.patch.dict( 'bodhi.server.consumers.masher.config', {'fedora_testing_master_repomd': @@ -2038,8 +2028,8 @@ def test_checksum_match_immediately(self, urlopen, publish): """ Assert correct operation when the repomd checksum matches immediately. """ - t = MasherThread(self._make_msg()['body']['msg']['composes'][0], - 'bowlofeggs', log, self.Session, self.tempdir) + t = PungiComposerThread(self._make_msg()['body']['msg']['composes'][0], + 'bowlofeggs', log, self.Session, self.tempdir) t.compose = self.db.query(Compose).one() t.id = 'f26-updates-testing' t.path = os.path.join(self.tempdir, t.id + '-' + time.strftime("%y%m%d.%H%M")) @@ -2049,7 +2039,7 @@ def test_checksum_match_immediately(self, urlopen, publish): with open(os.path.join(repodata, 'repomd.xml'), 'w') as repomd: repomd.write('---\nyaml: rules') - t.wait_for_sync() + t._wait_for_sync() expected_calls = [ mock.call(topic='mashtask.sync.wait', msg={'repo': t.id, 'agent': 'bowlofeggs'}, @@ -2081,8 +2071,8 @@ def test_no_checkarch(self, urlopen, publish): """ Assert error when no checkarch is found. """ - t = MasherThread(self._make_msg()['body']['msg']['composes'][0], - 'bowlofeggs', log, self.Session, self.tempdir) + t = PungiComposerThread(self._make_msg()['body']['msg']['composes'][0], + 'bowlofeggs', log, self.Session, self.tempdir) t.compose = self.db.query(Compose).one() t.id = 'f26-updates-testing' t.path = os.path.join(self.tempdir, t.id + '-' + time.strftime("%y%m%d.%H%M")) @@ -2093,10 +2083,10 @@ def test_no_checkarch(self, urlopen, publish): repomd.write('---\nyaml: rules') try: - t.wait_for_sync() + t._wait_for_sync() assert False, "Compose with just source passed" except Exception as ex: - assert str(ex) == "Not found an arch to wait_for_sync with" + assert str(ex) == "Not found an arch to _wait_for_sync with" @mock.patch.dict( 'bodhi.server.consumers.masher.config', @@ -2111,8 +2101,8 @@ def test_checksum_match_third_try(self, urlopen, sleep, publish): """ Assert correct operation when the repomd checksum matches on the third try. """ - t = MasherThread(self._make_msg()['body']['msg']['composes'][0], - 'bowlofeggs', log, self.Session, self.tempdir) + t = PungiComposerThread(self._make_msg()['body']['msg']['composes'][0], + 'bowlofeggs', log, self.Session, self.tempdir) t.compose = self.db.query(Compose).one() t.id = 'f26-updates-testing' t.path = os.path.join(self.tempdir, t.id + '-' + time.strftime("%y%m%d.%H%M")) @@ -2122,7 +2112,7 @@ def test_checksum_match_third_try(self, urlopen, sleep, publish): with open(os.path.join(repodata, 'repomd.xml'), 'w') as repomd: repomd.write('---\nyaml: rules') - t.wait_for_sync() + t._wait_for_sync() expected_calls = [ mock.call(topic='mashtask.sync.wait', msg={'repo': t.id, 'agent': 'bowlofeggs'}, @@ -2155,8 +2145,8 @@ def test_httperror(self, urlopen, sleep, publish): """ Assert that an HTTPError is properly caught and logged, and that the algorithm continues. """ - t = MasherThread(self._make_msg()['body']['msg']['composes'][0], - 'bowlofeggs', log, self.Session, self.tempdir) + t = PungiComposerThread(self._make_msg()['body']['msg']['composes'][0], + 'bowlofeggs', log, self.Session, self.tempdir) t.compose = self.db.query(Compose).one() t.id = 'f26-updates-testing' t.log = mock.MagicMock() @@ -2167,7 +2157,7 @@ def test_httperror(self, urlopen, sleep, publish): with open(os.path.join(repodata, 'repomd.xml'), 'w') as repomd: repomd.write('---\nyaml: rules') - t.wait_for_sync() + t._wait_for_sync() expected_calls = [ mock.call(topic='mashtask.sync.wait', msg={'repo': t.id, 'agent': 'bowlofeggs'}, @@ -2199,8 +2189,8 @@ def test_missing_config_key(self, publish): """ Assert that a ValueError is raised when the needed *_master_repomd config is missing. """ - t = MasherThread(self._make_msg()['body']['msg']['composes'][0], - 'bowlofeggs', log, self.Session, self.tempdir) + t = PungiComposerThread(self._make_msg()['body']['msg']['composes'][0], + 'bowlofeggs', log, self.Session, self.tempdir) t.compose = self.db.query(Compose).one() t.id = 'f26-updates-testing' t.path = os.path.join(self.tempdir, t.id + '-' + time.strftime("%y%m%d.%H%M")) @@ -2211,7 +2201,7 @@ def test_missing_config_key(self, publish): repomd.write('---\nyaml: rules') with self.assertRaises(ValueError) as exc: - t.wait_for_sync() + t._wait_for_sync() self.assertEqual(six.text_type(exc.exception), 'Could not find fedora_testing_master_repomd in the config file') @@ -2227,8 +2217,8 @@ def test_missing_repomd(self, publish): """ Assert that an error is logged when the local repomd is missing. """ - t = MasherThread(self._make_msg()['body']['msg']['composes'][0], - 'bowlofeggs', log, self.Session, self.tempdir) + t = PungiComposerThread(self._make_msg()['body']['msg']['composes'][0], + 'bowlofeggs', log, self.Session, self.tempdir) t.compose = self.db.query(Compose).one() t.id = 'f26-updates-testing' t.log = mock.MagicMock() @@ -2236,7 +2226,7 @@ def test_missing_repomd(self, publish): repodata = os.path.join(t.path, 'compose', 'Everything', 'x86_64', 'os', 'repodata') os.makedirs(repodata) - t.wait_for_sync() + t._wait_for_sync() publish.assert_called_once_with(topic='mashtask.sync.wait', msg={'repo': t.id, 'agent': 'bowlofeggs'}, force=True) @@ -2257,8 +2247,8 @@ def test_urlerror(self, urlopen, sleep, publish): """ Assert that a URLError is properly caught and logged, and that the algorithm continues. """ - t = MasherThread(self._make_msg()['body']['msg']['composes'][0], - 'bowlofeggs', log, self.Session, self.tempdir) + t = PungiComposerThread(self._make_msg()['body']['msg']['composes'][0], + 'bowlofeggs', log, self.Session, self.tempdir) t.compose = self.db.query(Compose).one() t.id = 'f26-updates-testing' t.log = mock.MagicMock() @@ -2269,7 +2259,7 @@ def test_urlerror(self, urlopen, sleep, publish): with open(os.path.join(repodata, 'repomd.xml'), 'w') as repomd: repomd.write('---\nyaml: rules') - t.wait_for_sync() + t._wait_for_sync() expected_calls = [ mock.call(topic='mashtask.sync.wait', msg={'repo': t.id, 'agent': 'bowlofeggs'}, @@ -2290,15 +2280,15 @@ def test_urlerror(self, urlopen, sleep, publish): sleep.assert_called_once_with(200) -class TestMasherThread__mark_status_changes(MasherThreadBaseTestCase): +class TestComposerThread__mark_status_changes(ComposerThreadBaseTestCase): """Test the _mark_status_changes() method.""" def test_stable_update(self): """Assert that a stable update gets the right status.""" update = Update.query.one() update.status = UpdateStatus.testing update.request = UpdateRequest.stable - t = MasherThread(self._make_msg()['body']['msg']['composes'][0], - 'bowlofeggs', log, self.Session, self.tempdir) + t = ComposerThread(self._make_msg()['body']['msg']['composes'][0], + 'bowlofeggs', log, self.Session, self.tempdir) t.compose = self.db.query(Compose).one() t._mark_status_changes() @@ -2319,8 +2309,8 @@ def test_testing_update(self): update = Update.query.one() update.status = UpdateStatus.pending update.request = UpdateRequest.testing - t = MasherThread(self._make_msg()['body']['msg']['composes'][0], - 'bowlofeggs', log, self.Session, self.tempdir) + t = ComposerThread(self._make_msg()['body']['msg']['composes'][0], + 'bowlofeggs', log, self.Session, self.tempdir) t.compose = self.db.query(Compose).one() t._mark_status_changes() @@ -2337,14 +2327,51 @@ def test_testing_update(self): self.assertTrue(update.pushed) -class TestMasherThread__unlock_updates(MasherThreadBaseTestCase): +class TestComposerThread_send_notifications(ComposerThreadBaseTestCase): + """Test ComposerThread.send_notifications.""" + + @mock.patch('bodhi.server.consumers.masher.notifications.publish') + def test_getlogin_raising_oserror(self, publish): + """Assert that "masher" is used as the agent if getlogin() raises OSError.""" + t = ComposerThread(self._make_msg()['body']['msg']['composes'][0], + 'bowlofeggs', log, self.Session, self.tempdir) + t.compose = self.db.query(Compose).one() + + with mock.patch('bodhi.server.consumers.masher.os.getlogin', side_effect=OSError()): + t.send_notifications() + + # The agent should be "masher" since OSError was raised. + self.assertEqual(publish.mock_calls[0][2]['msg']['agent'], 'masher') + + +class TestComposerThread_send_testing_digest(ComposerThreadBaseTestCase): + """Test ComposerThread.send_testing_digest().""" + + @mock.patch('bodhi.server.consumers.masher.log.warn') + def test_test_list_not_configured(self, warn): + """If a test_announce_list setting is not found, a warning should be logged.""" + t = ComposerThread(self._make_msg()['body']['msg']['composes'][0], + 'bowlofeggs', log, self.Session, self.tempdir) + t.compose = self.db.query(Compose).one() + t.testing_digest = {'Fedora 17': {'fake': 'content'}} + t._checkpoints = {} + t.db = self.Session + + with mock.patch.dict(config, {'fedora_test_announce_list': None}): + t.send_testing_digest() + + warn.assert_called_once_with( + '%r undefined. Not sending updates-testing digest', 'fedora_test_announce_list') + + +class TestComposerThread__unlock_updates(ComposerThreadBaseTestCase): """Test the _unlock_updates() method.""" def test__unlock_updates(self): """Assert that _unlock_updates() works correctly.""" update = Update.query.one() update.request = UpdateRequest.testing - t = MasherThread(self._make_msg()['body']['msg']['composes'][0], - 'bowlofeggs', log, self.Session, self.tempdir) + t = ComposerThread(self._make_msg()['body']['msg']['composes'][0], + 'bowlofeggs', log, self.Session, self.tempdir) t.compose = self.db.query(Compose).one() t._unlock_updates() @@ -2352,3 +2379,110 @@ def test__unlock_updates(self): update = Update.query.one() self.assertIsNone(update.request) self.assertFalse(update.locked) + + +class TestPungiComposerThread__punge(ComposerThreadBaseTestCase): + """Test the PungiComposerThread._punge() method.""" + + @mock.patch('bodhi.server.consumers.masher.subprocess.Popen') + def test_skips_if_path_defined(self, Popen): + """_punge should log a message and skip running if path is truthy.""" + t = PungiComposerThread(self._make_msg()['body']['msg']['composes'][0], + 'bowlofeggs', log, self.Session, self.tempdir) + t.log.info = mock.MagicMock() + t.path = '/some/path' + + t._punge() + + t.log.info.assert_called_once_with('Skipping completed repo: %s', '/some/path') + # Popen() should not have been called since we should have skipping running pungi. + self.assertEqual(Popen.call_count, 0) + + +class TestPungiComposerThread__stage_repo(ComposerThreadBaseTestCase): + """Test PungiComposerThread._stage_repo().""" + + def test_old_link_present(self): + """If a link from the last run is still present, no error should be raised.""" + t = PungiComposerThread(self._make_msg()['body']['msg']['composes'][0], + 'ralph', log, self.Session, self.tempdir) + t.id = 'f17-updates-testing' + t.log.info = mock.MagicMock() + t.path = os.path.join(self.tempdir, 'latest-f17-updates-testing') + stage_dir = os.path.join(self.tempdir, 'stage_dir') + os.makedirs(t.path) + os.mkdir(stage_dir) + link = os.path.join(stage_dir, t.id) + os.symlink(t.path, link) + + with mock.patch.dict(config, {'mash_stage_dir': stage_dir}): + t._stage_repo() + + self.assertTrue(os.path.islink(link)) + self.assertEqual(os.readlink(link), t.path) + self.assertEqual( + t.log.info.mock_calls, + [mock.call('Creating symlink: %s => %s' % (link, t.path))]) + + def test_stage_dir_de(self): + """Test for when stage_dir does exist.""" + t = PungiComposerThread(self._make_msg()['body']['msg']['composes'][0], + 'ralph', log, self.Session, self.tempdir) + t.id = 'f17-updates-testing' + t.log.info = mock.MagicMock() + t.path = os.path.join(self.tempdir, 'latest-f17-updates-testing') + stage_dir = os.path.join(self.tempdir, 'stage_dir') + os.makedirs(t.path) + os.mkdir(stage_dir) + + with mock.patch.dict(config, {'mash_stage_dir': stage_dir}): + t._stage_repo() + + link = os.path.join(stage_dir, t.id) + self.assertTrue(os.path.islink(link)) + self.assertEqual(os.readlink(link), t.path) + self.assertEqual( + t.log.info.mock_calls, + [mock.call('Creating symlink: %s => %s' % (link, t.path))]) + + def test_stage_dir_dne(self): + """Test for when stage_dir does not exist.""" + t = PungiComposerThread(self._make_msg()['body']['msg']['composes'][0], + 'ralph', log, self.Session, self.tempdir) + t.id = 'f17-updates-testing' + t.log.info = mock.MagicMock() + t.path = os.path.join(self.tempdir, 'latest-f17-updates-testing') + stage_dir = os.path.join(self.tempdir, 'stage_dir') + os.makedirs(t.path) + + with mock.patch.dict(config, {'mash_stage_dir': stage_dir}): + t._stage_repo() + + link = os.path.join(stage_dir, t.id) + self.assertTrue(os.path.islink(link)) + self.assertEqual(os.readlink(link), t.path) + self.assertEqual( + t.log.info.mock_calls, + [mock.call('Creating mash_stage_dir %s', stage_dir), + mock.call('Creating symlink: %s => %s' % (link, t.path))]) + + +class TestPungiComposerThread__wait_for_pungi(ComposerThreadBaseTestCase): + """Test PungiComposerThread._wait_for_pungi().""" + + def test_pungi_process_None(self): + """If pungi_process is None, a log should be written and the method should return.""" + t = PungiComposerThread(self._make_msg()['body']['msg']['composes'][0], + 'ralph', log, self.Session, self.tempdir) + t.compose = self.db.query(Compose).one() + t.db = self.Session + t.log.info = mock.MagicMock() + t._checkpoints = {} + + t._wait_for_pungi(None) + + self.assertEqual( + t.log.info.mock_calls, + [mock.call('Compose object updated.'), + mock.call('Not waiting for pungi thread, as there was no pungi')]) + self.assertEqual(t.compose.state, ComposeState.punging)