diff --git a/build-release b/build-release index eef32aa23f341..dc61862db7b95 100755 --- a/build-release +++ b/build-release @@ -5,12 +5,20 @@ # ARCHITECTURE: architecture for the build # ISSUE_NUMBER: the number of the issue that triggered the build # DRY_RUN_PARAM: parameter that to use dry-run +# ONLY_BUILD_TOOLCONF: if True, tells me to only build cmssw-tool-conf HERE=$WORKSPACE/build BUILD_DIR=$HERE/$CMSSW_X_Y_Z-build mkdir -p $BUILD_DIR QUEUE=$CMSSW_QUEUE +if [ "X$ONLY_BUILD_TOOLCONF" = Xtrue ]; then + POST_ACTION='TOOL_CONF_BUILDING' +else + POST_ACTION='BUILDING' +fi + + git clone git@github.com:cms-sw/cmsdist.git $BUILD_DIR/CMSDIST pushd $BUILD_DIR/CMSDIST eval $(cat $WORKSPACE/config.map | grep "SCRAM_ARCH=$ARCHITECTURE;" | grep "RELEASE_QUEUE=$QUEUE;") @@ -31,7 +39,7 @@ git clone -b $PKGTOOLS_TAG git@github.com:cms-sw/pkgtools.git $BUILD_DIR/PKGTOOL BRANCH_DETAILS="CMSDIST Branch: [$CMSDIST_TAG](https://github.com/cms-sw/cmsdist/commits/$CMSDIST_TAG), PKGTOOLS Branch: [$PKGTOOLS_TAG](https://github.com/cms-sw/pkgtools/commits/$PKGTOOLS_TAG)" WORKER=`hostname -s` -./report-build-release-status ${BUILD_NUMBER} $WORKER $ISSUE_NUMBER $ARCHITECTURE $CMSSW_X_Y_Z BUILDING $DRY_RUN_PARAM -d "$BRANCH_DETAILS" +./report-build-release-status ${BUILD_NUMBER} $WORKER $ISSUE_NUMBER $ARCHITECTURE $CMSSW_X_Y_Z $POST_ACTION $DRY_RUN_PARAM -d "$BRANCH_DETAILS" pushd $BUILD_DIR # prepare cmssw.spec @@ -53,6 +61,19 @@ pushd $BUILD_DIR TOOL_CONF_PACKAGES=`grep '^%define \(build\|\)directpkgreqs' SPECS/cms/cmssw$PATCH/*/spec | grep -v '%{nil}' | sed 's|.*directpkgreqs[ \t]*||' | tr ' ' '\n' | cut -f2 -d/ | sort | uniq | tr '\n' ' '` $CMSBUILD_CMD build $TOOL_CONF_PACKAGES $CMSBUILD_CMD --sync-back upload $TOOL_CONF_PACKAGES + + if [ "X$ONLY_BUILD_TOOLCONF" = Xtrue ]; then + echo 'I was asked to build only cmssw-tool-conf, exiting without building cmssw' + + pushd $HERE/$CMSSW_X_Y_Z-build/CMSDIST + git tag REL/$CMSSW_X_Y_Z-tool-conf/$ARCHITECTURE + git push origin REL/$CMSSW_X_Y_Z-tool-conf/$ARCHITECTURE + popd + # If it reaches here it is because there were no errors + echo 'ALL_OK' + exit 0 + fi + $CMSBUILD_CMD deprecate-local $TOOL_CONF_PACKAGES $CMSBUILD_CMD build cmssw$PATCH popd diff --git a/process-build-release-request b/process-build-release-request index 549b2bae8b23c..3e80897d6e2c1 100755 --- a/process-build-release-request +++ b/process-build-release-request @@ -34,14 +34,21 @@ RELEASE_CREATION_FAIL_MSG = 'There was an error while attempting to create {rel_ WRONG_RELEASE_NAME_MSG = 'The release name is malformed. Please check for typos.' ACK_MSG = 'Request received. I will start to build the release after one of the following approve ' \ 'the issue: {approvers_list}. You can do this by writing "+1" in a ' \ - 'comment.\n' + 'comment.\n You can also ask me to begin to build cmssw-tool-conf first ( Cannot be done for patch releases ). To do this write ' \ + '"build cmssw-tool-conf" in a comment. I will start to build cmssw-tool-conf and then wait for the "+1" ' \ + 'to start the build of the release. \n' WATCHERS_MSG = '{watchers_list} you requested to watch the automated builds for {queue}' QUEUING_BUILDS_MSG = 'Queuing Jenkins build for the following architectures: %s \n' \ 'You can abort the build by writing "Abort" in a comment. I will delete the release, ' \ 'the cmssw and cmsdist tag, and close the issue. You can\'t abort the upload once at' \ ' least one achitecture is being uploaded' +QUEUING_TOOLCONF_MSG = 'Queuing Jenkins build for cmssw-tool-conf for the following architectures: %s \n' \ + 'Be aware that I am building only cmssw-tool-conf. You still need to "+1" this issue to ' \ + 'make me start the build of the release. For each architecture, I will only start to build ' \ + 'the release after cmssw-tool-conf finishes building.' QUEING_UPLOADS_MSG = 'Queing Jenkins upload for {architecture}' CLEANUP_STARTED_MSG = 'The cleanup has started for {architecture}' +NOT_TOOLCONF_FOR_PATCH_MSG = 'You cannot ask me to build cmssw-tool-conf for patch releases. Please delete that message.' BUILD_QUEUED_LABEL = 'build-release-queued' BUILD_STARTED_LABEL = 'build-release-started' JENKINS_CMSSW_X_Y_Z = 'CMSSW_X_Y_Z' @@ -49,6 +56,7 @@ JENKINS_ARCH = 'ARCHITECTURE' JENKINS_ISSUE_NUMBER = 'ISSUE_NUMBER' JENKINS_MACHINE_NAME = 'MACHINE_NAME' JENKINS_CMSSW_QUEUE = 'CMSSW_QUEUE' +JENKINS_ONLY_TOOL_CONF = 'ONLY_BUILD_TOOLCONF' WRONG_NOTES_RELEASE_MSG = '{previous_release} does not appear to be a valid release name' GENERATING_RELEASE_NOTES_MSG = 'Generating release notes since {previous_release}. \n' \ 'You can see the progress here: \n' \ @@ -58,6 +66,8 @@ REL_NAME_REGEXP="(CMSSW_[0-9]+_[0-9]+)_[0-9]+(_SLHC[0-9]*|)(_pre[0-9]+|_[a-zA-Z] UPLOAD_COMMENT = 'upload %s' UPLOAD_ALL_COMMENT = '[uU]pload all' ABORT_COMMENT = '^[Aa]bort$' +BUILD_TOOLCONF = '^[Bb]uild cmssw-tool-conf' +APPROVAL_COMMENT = '^[+]1$' DEFAULT_CHECK_COMMENT = ( REQUEST_BUILD_RELEASE + [ 'cmsbuild' ] ) RELEASE_NOTES_GENERATED_LBL = 'release-notes-requested' JENKINS_PREV_RELEASE='PREVIOUS_RELEASE' @@ -82,6 +92,8 @@ BUILD_IN_PROGRESS = 'build-in-progress' BUILD_STARTED = 'build-started' # The build has been aborted. BUILD_ABORTED = 'build-aborted' +# they requested to build cmssw-tool-conf and it is being built +TOOLCONF_BUILDING = 'toolconf-building' # ------------------------------------------------------------------------------- # Functions @@ -186,8 +198,9 @@ def check_if_already_processed( issue ): return False # # Creates the properties files to trigger the build in Jenkins +# if only_toolconf is selected, it adds a parameter to tell the script to only build cmssw-tool-conf # -def create_properties_files(release_name, architectures, issue_number, queue): +def create_properties_files(release_name, architectures, issue_number, queue, only_toolconf=False): if opts.dryRun: print 'Not creating properties files for (dry-run): %s' % ", ".join( architectures ) return @@ -200,6 +213,8 @@ def create_properties_files(release_name, architectures, issue_number, queue): out_file.write( '%s=%s\n' % ( JENKINS_ARCH , arch ) ) out_file.write( '%s=%s\n' % ( JENKINS_ISSUE_NUMBER , issue_number ) ) out_file.write( '%s=%s\n' % ( JENKINS_CMSSW_QUEUE , queue) ) + out_file.write( '%s=%s\n' % ( JENKINS_ONLY_TOOL_CONF, only_toolconf ) ) + # # generates the properties file for triggering the release notes @@ -383,6 +398,8 @@ def get_issue_status( issue ): return PENDING_APPROVAL if BUILD_IN_PROGRESS in labels: return BUILD_IN_PROGRESS + if TOOLCONF_BUILDING in labels: + return TOOLCONF_BUILDING # # removes the labels of the issue @@ -429,6 +446,10 @@ def fillDeatilsArchsLists( issue ): UPLOAD_OK.extend( [ x.split('-')[0] for x in labels if '-upload-ok' in x ] ) UPLOADING.extend( [ x.split('-')[0] for x in labels if '-uploading' in x ] ) BUILD_ERROR.extend( [ x.split('-')[0] for x in labels if '-build-error' in x ] ) + TOOL_CONF_BUILDING.extend( [ x.split('-')[0] for x in labels if '-tool-conf-building' in x ] ) + TOOL_CONF_OK.extend( [ x.split('-')[0] for x in labels if '-tool-conf-ok' in x ] ) + TOOL_CONF_ERROR.extend( [ x.split('-')[0] for x in labels if '-tool-conf-error' in x ] ) + TOOL_CONF_WAITING.extend( [ x.split('-')[0] for x in labels if '-tool-conf-waiting' in x ] ) TO_CLEANUP.extend( UPLOAD_OK + BUILD_ERROR + TO_UPLOAD ) # @@ -456,6 +477,40 @@ def triggerCleanup( issue, comments, release_name ): remove_label( issue, arch + '-build-error' ) add_label( issue, arch + '-finished' ) +# +# Creates the release in github, including the cmssw tag. It then creates the files to trigger the builds in jenkins +#TODO check cmssw toolconf +# +def start_release_build( issue, release_name, release_branch, architectures ): + + # if someone approved, go ahead and create the release + cmssw_repo = gh.get_organization( GH_CMSSW_ORGANIZATION ).get_repo( GH_CMSSW_REPO ) + release_created = create_release_github( cmssw_repo, release_name, release_branch ) + if not release_created: + msg = RELEASE_CREATION_FAIL_MSG.format( rel_name=release_name ) + post_message( issue , RELEASE_CREATION_FAIL_MSG.format( rel_name=release_name ) ) + exit( 0 ) + + msg = RELEASE_CREATED_MSG % ( RELEASE_BASE_URL % release_name ) + post_message( issue , msg ) + + ready_to_build = list( set( architectures ) - set( TOOL_CONF_WAITING ) - set( TOOL_CONF_ERROR ) - set( TOOL_CONF_BUILDING ) ) + create_properties_files( release_name, ready_to_build, issue.number, release_queue ) + msg = QUEUING_BUILDS_MSG % ', '.join( architectures ) + post_message( issue , msg ) + +# +# Creates the files to trigger the build of cmssw-tool-conf in jenkins. +# +def start_tool_conf_build( issue, release_name, release_branch, architectures ): + + create_properties_files( release_name, architectures, issue.number, release_queue, only_toolconf=True ) + QUEUING_TOOLCONF_MSG + msg = QUEUING_TOOLCONF_MSG % ', '.join( architectures ) + post_message( issue , msg ) + remove_labels( issue ) + add_label( issue, TOOLCONF_BUILDING ) + # ------------------------------------------------------------------------------- # Start of execution @@ -546,6 +601,10 @@ if __name__ == "__main__": UPLOADING = [] BUILD_ERROR = [] TO_CLEANUP = [] + TOOL_CONF_BUILDING = [] + TOOL_CONF_OK = [] + TOOL_CONF_ERROR = [] + TOOL_CONF_WAITING = [] # These lists are filled by fillDeatilsArchsLists( issue ) fillDeatilsArchsLists( issue ) @@ -579,28 +638,49 @@ if __name__ == "__main__": if status == PENDING_APPROVAL: approval_comments = search_in_comments( comments, APPROVE_BUILD_RELEASE , '^[+]1$', True ) - if not approval_comments: - print 'Build not approved yet' + build_toolconf_commments = search_in_comments( comments, APPROVE_BUILD_RELEASE , BUILD_TOOLCONF, True ) + is_patch = 'patch' in release_name + + if build_toolconf_commments: + if is_patch: + post_message( issue, NOT_TOOLCONF_FOR_PATCH_MSG ) + else: + start_tool_conf_build( issue, release_name, release_branch, architectures ) + elif approval_comments: + start_release_build( issue, release_name, release_branch, architectures ) + remove_labels( issue ) + add_label( issue, BUILD_IN_PROGRESS ) + else: + print 'Build not approved yet, or cmssw-tool-conf requested' exit( 0 ) - # if someone approved, go ahead and create the release - cmssw_repo = gh.get_organization( GH_CMSSW_ORGANIZATION ).get_repo( GH_CMSSW_REPO ) - release_created = create_release_github( cmssw_repo, release_name, release_branch ) - if not release_created: - msg = RELEASE_CREATION_FAIL_MSG.format( rel_name=release_name ) - post_message( issue , RELEASE_CREATION_FAIL_MSG.format( rel_name=release_name ) ) - exit( 0 ) + if status == TOOLCONF_BUILDING: + print 'Waiting for approval to start the build' + approval_comments = search_in_comments( comments, APPROVE_BUILD_RELEASE , APPROVAL_COMMENT, True ) + if approval_comments: + print 'Build approved, switching to "Build in Progress" state' + + # add a label for each arch for which tool conf has not started in jenkins + tool_conf_reported = ( TOOL_CONF_BUILDING + TOOL_CONF_OK + TOOL_CONF_ERROR ) + not_started = list( set( architectures ) - set( tool_conf_reported ) ) - msg = RELEASE_CREATED_MSG % ( RELEASE_BASE_URL % release_name ) - post_message( issue , msg ) + for arch in not_started: + add_label( issue, arch + '-tool-conf-waiting' ) + TOOL_CONF_WAITING.append( arch ) - create_properties_files( release_name, architectures, issue_id, release_queue ) - msg = QUEUING_BUILDS_MSG % ', '.join( architectures ) - post_message( issue , msg ) - remove_labels( issue ) - add_label( issue, BUILD_IN_PROGRESS ) + remove_label( issue, TOOLCONF_BUILDING ) + add_label( issue, BUILD_IN_PROGRESS ) + start_release_build( issue, release_name, release_branch, architectures ) if status == BUILD_IN_PROGRESS: + + ready_to_build = list( set( architectures ) - set( TOOL_CONF_WAITING ) - set( TOOL_CONF_ERROR ) - set( TOOL_CONF_BUILDING ) ) + print 'conf tool waiting' + print TOOL_CONF_WAITING + print 'ready to build' + print ready_to_build + create_properties_files( release_name, ready_to_build, issue.number, release_queue ) + # the build can be aborted only if none of the architectures has been uploaded or is uploading if not ( UPLOADING + UPLOAD_OK ): abort_comments = search_in_comments( comments , APPROVE_BUILD_RELEASE , ABORT_COMMENT, True ) @@ -671,6 +751,3 @@ if __name__ == "__main__": add_label( issue, arch + '-uploading' ) exit( 0 ) - - - diff --git a/report-build-release-status b/report-build-release-status index e3ac0d2ffaf8c..f2763f726f251 100755 --- a/report-build-release-status +++ b/report-build-release-status @@ -15,7 +15,10 @@ from os.path import expanduser GH_CMSSW_ORGANIZATION = 'cms-sw' GH_CMSSW_REPO = 'cmssw' POST_BUILDING='BUILDING' +POST_TOOL_CONF_BUILDING='TOOL_CONF_BUILDING' BUILD_OK='BUILD_OK' +TOOL_CONF_OK='TOOL_CONF_OK' +TOOL_CONF_ERROR = 'TOOL_CONF_ERROR' BUILD_ERROR='BUILD_ERROR' UPLOADING='UPLOADING' UPLOAD_OK='UPLOAD_OK' @@ -28,13 +31,23 @@ TESTS_OK='TESTS_OK' TESTS_ERROR='TESTS_ERROR' BUILDING_MSG='The build has started for {architecture} in {machine}. \n' \ 'You can see the progress here: https://cmssdt.cern.ch/jenkins/job/build-release/{jk_build_number}/console \n' \ - '{details}' + '{details}' +BUILDING_TOOL_CONF_MSG='The cmssw-tool-conf build has started for {architecture} in {machine}. \n' \ + 'You can see the progress here: https://cmssdt.cern.ch/jenkins/job/build-release/{jk_build_number}/console \n' \ + '{details}' BUILD_OK_MSG='The build has finished sucessfully for the architecture {architecture} and is ready to be uploaded. \n' \ '* You can start the uploads by writing the comment: "upload all". I will upload all the architectures as soon as the build finishes successfully.\n' \ '* Or you can start the upload only for {architecture} by writing: "upload {architecture}".\n \n' \ 'You can see the log for the build here: \n' \ '{log_url} \n' \ 'Some tests ( runTheMatrix.py -s ) are being run, the results will be posted when done.' +TOOL_CONF_OK_MSG='The cmssw-tool-conf build has finished sucessfully for the architecture {architecture} and it was automatically uploaded. \n' \ + 'Remember that if you write "+1" I will start to build this and all the architectures as soon as their cmssw-tool-conf finish.\n' \ + 'You can see the log for the build here: \n' \ + '{log_url} \n' +TOOL_CONF_ERROR_MSG='There was an error building cmssw-tool-conf for {architecture} \n' \ + 'You can see the log for the build here: \n' \ + '{log_url} \n' UPLOADING_MSG='The upload has started for {architecture} in {machine}. \n' \ 'You can see the progress here: https://cmssdt.cern.ch/jenkins/job/upload-release/{jk_build_number}/console' UPLOAD_OK_MSG='The upload has successfully finished for {architecture} \n You can see the log here: \n {log_url} \n' \ @@ -51,7 +64,7 @@ TESTS_ERROR_MSG='There was an error when running the tests for {architecture} \n BUILD_ERROR_MSG='The was an error for {architecture}. \n You can see the log here: \n {log_url}' BUILD_QUEUED_LABEL = 'build-release-queued' BUILD_STARTED = 'build-release-started' -BASE_BUILD_LOG_URL = 'https://cmssdt.cern.ch/SDT/jenkins-artifacts/auto-build-release/%s-%s/%d' +BASE_BUILD_LOG_URL = 'https://cmssdt.cern.ch/SDT/jenkins-artifacts/auto-build-release/%s-%s/%d' BASE_UPLOAD_LOG_URL = 'https://cmssdt.cern.ch/SDT/jenkins-artifacts/auto-upload-release/%s-%s/%d' BASE_CLEANUP_LOG_URL = 'https://cmssdt.cern.ch/SDT/jenkins-artifacts/cleanup-auto-build/%s-%s/%d' @@ -131,6 +144,17 @@ if __name__ == "__main__": new_label = arch+'-building' add_label( issue, new_label ) + elif action == POST_TOOL_CONF_BUILDING: + + msg_details = '' + if opts.details: + msg_details = opts.details + msg = BUILDING_TOOL_CONF_MSG.format( architecture=arch, machine=hostname, jk_build_number=jenkins_build_number, details=msg_details ) + post_message( issue , msg ) + remove_label( issue, arch+'-tool-conf-waiting' ) + new_label = arch+'-tool-conf-building' + add_label( issue, new_label ) + elif action == BUILD_OK: results_url = BASE_BUILD_LOG_URL % (release_name,arch,jenkins_build_number) @@ -139,6 +163,14 @@ if __name__ == "__main__": remove_label( issue, arch+'-building' ) add_label( issue, arch+'-build-ok' ) + elif action == TOOL_CONF_OK: + + results_url = BASE_BUILD_LOG_URL % (release_name,arch,jenkins_build_number) + msg = TOOL_CONF_OK_MSG.format( architecture=arch, log_url=results_url ) + post_message( issue , msg ) + remove_label( issue, arch+'-tool-conf-building' ) + add_label( issue, arch+'-tool-conf-ok' ) + elif action == BUILD_ERROR: results_url = BASE_BUILD_LOG_URL % (release_name,arch,jenkins_build_number) @@ -147,6 +179,14 @@ if __name__ == "__main__": remove_label( issue, arch+'-building' ) add_label( issue, arch+'-build-error' ) + elif action == TOOL_CONF_ERROR: + + results_url = BASE_BUILD_LOG_URL % (release_name,arch,jenkins_build_number) + msg = TOOL_CONF_ERROR_MSG.format( architecture=arch, log_url=results_url ) + post_message( issue , msg ) + remove_label( issue, arch+'-tool-conf-building' ) + add_label( issue, arch+'-tool-conf-error' ) + elif action == UPLOADING: msg = UPLOADING_MSG.format( architecture=arch, machine=hostname, jk_build_number=jenkins_build_number)