Merge pull request cms-sw#362 from nclopezo/allow-build-tool-conf

AutoBuilds: Allow to build cmssw-tool-conf before starting cmssw
makortel · Mar 4, 2015 · 1e687d7 · 1e687d7
2 parents d791f94 + fc08f7f
commit 1e687d7
Show file tree

Hide file tree

Showing 3 changed files with 162 additions and 24 deletions.
diff --git a/build-release b/build-release
@@ -5,12 +5,20 @@
 # ARCHITECTURE: architecture for the build
 # ISSUE_NUMBER: the number of the issue that triggered the build
 # DRY_RUN_PARAM: parameter that to use dry-run
+# ONLY_BUILD_TOOLCONF: if True, tells me to only build cmssw-tool-conf 
 
 HERE=$WORKSPACE/build
 BUILD_DIR=$HERE/$CMSSW_X_Y_Z-build
 mkdir -p $BUILD_DIR
 QUEUE=$CMSSW_QUEUE
 
+if [ "X$ONLY_BUILD_TOOLCONF" = Xtrue ]; then
+  POST_ACTION='TOOL_CONF_BUILDING'
+else
+  POST_ACTION='BUILDING'
+fi
+
+
 git clone [email protected]:cms-sw/cmsdist.git $BUILD_DIR/CMSDIST
 pushd $BUILD_DIR/CMSDIST
   eval $(cat $WORKSPACE/config.map | grep "SCRAM_ARCH=$ARCHITECTURE;" | grep "RELEASE_QUEUE=$QUEUE;")
@@ -31,7 +39,7 @@ git clone -b $PKGTOOLS_TAG [email protected]:cms-sw/pkgtools.git $BUILD_DIR/PKGTOOL
 
 BRANCH_DETAILS="CMSDIST Branch: [$CMSDIST_TAG](https://github.com/cms-sw/cmsdist/commits/$CMSDIST_TAG), PKGTOOLS Branch: [$PKGTOOLS_TAG](https://github.com/cms-sw/pkgtools/commits/$PKGTOOLS_TAG)"
 WORKER=`hostname -s`
-./report-build-release-status ${BUILD_NUMBER} $WORKER $ISSUE_NUMBER $ARCHITECTURE $CMSSW_X_Y_Z BUILDING $DRY_RUN_PARAM -d "$BRANCH_DETAILS"
+./report-build-release-status ${BUILD_NUMBER} $WORKER $ISSUE_NUMBER $ARCHITECTURE $CMSSW_X_Y_Z $POST_ACTION $DRY_RUN_PARAM -d "$BRANCH_DETAILS"
 
 pushd $BUILD_DIR
   # prepare cmssw.spec
@@ -53,6 +61,19 @@ pushd $BUILD_DIR
   TOOL_CONF_PACKAGES=`grep '^%define \(build\|\)directpkgreqs' SPECS/cms/cmssw$PATCH/*/spec | grep -v '%{nil}' | sed 's|.*directpkgreqs[ \t]*||' | tr ' ' '\n' | cut -f2 -d/ | sort | uniq | tr '\n' ' '`
   $CMSBUILD_CMD build $TOOL_CONF_PACKAGES
   $CMSBUILD_CMD --sync-back upload $TOOL_CONF_PACKAGES
+
+  if [ "X$ONLY_BUILD_TOOLCONF" = Xtrue ]; then
+    echo 'I was asked to build only cmssw-tool-conf, exiting without building cmssw'
+
+    pushd $HERE/$CMSSW_X_Y_Z-build/CMSDIST
+      git tag  REL/$CMSSW_X_Y_Z-tool-conf/$ARCHITECTURE
+      git push origin REL/$CMSSW_X_Y_Z-tool-conf/$ARCHITECTURE
+    popd
+    # If it reaches here it is because there were no errors
+    echo 'ALL_OK'
+    exit 0
+  fi
+
   $CMSBUILD_CMD deprecate-local $TOOL_CONF_PACKAGES
   $CMSBUILD_CMD build cmssw$PATCH
 popd

diff --git a/process-build-release-request b/process-build-release-request
@@ -34,21 +34,29 @@ RELEASE_CREATION_FAIL_MSG = 'There was an error while attempting to create {rel_
 WRONG_RELEASE_NAME_MSG = 'The release name is malformed. Please check for typos.'
 ACK_MSG = 'Request received. I will start to build the release after one of the following approve ' \
           'the issue: {approvers_list}. You can do this by writing "+1" in a ' \
-          'comment.\n'
+          'comment.\n You can also ask me to begin to build cmssw-tool-conf first ( Cannot be done for patch releases ). To do this write ' \
+          '"build cmssw-tool-conf" in a comment. I will start to build cmssw-tool-conf and then wait for the "+1" ' \
+          'to start the build of the release. \n' 
 WATCHERS_MSG = '{watchers_list} you requested to watch the automated builds for {queue}' 
 QUEUING_BUILDS_MSG = 'Queuing Jenkins build for the following architectures: %s \n' \
                      'You can abort the build by writing "Abort" in a comment. I will delete the release, ' \
                      'the cmssw and cmsdist tag, and close the issue. You can\'t abort the upload once at' \
                      ' least one achitecture is being uploaded'
+QUEUING_TOOLCONF_MSG = 'Queuing Jenkins build for cmssw-tool-conf for the following architectures: %s \n' \
+                     'Be aware that I am building only cmssw-tool-conf. You still need to "+1" this issue to ' \
+                     'make me start the build of the release. For each architecture, I will only start to build ' \
+                     'the release after cmssw-tool-conf finishes building.'
 QUEING_UPLOADS_MSG = 'Queing Jenkins upload for {architecture}'
 CLEANUP_STARTED_MSG = 'The cleanup has started for {architecture}'
+NOT_TOOLCONF_FOR_PATCH_MSG = 'You cannot ask me to build cmssw-tool-conf for patch releases. Please delete that message.'
 BUILD_QUEUED_LABEL = 'build-release-queued'
 BUILD_STARTED_LABEL = 'build-release-started'
 JENKINS_CMSSW_X_Y_Z = 'CMSSW_X_Y_Z'
 JENKINS_ARCH = 'ARCHITECTURE'
 JENKINS_ISSUE_NUMBER = 'ISSUE_NUMBER'
 JENKINS_MACHINE_NAME = 'MACHINE_NAME'
 JENKINS_CMSSW_QUEUE = 'CMSSW_QUEUE'
+JENKINS_ONLY_TOOL_CONF = 'ONLY_BUILD_TOOLCONF'
 WRONG_NOTES_RELEASE_MSG = '{previous_release} does not appear to be a valid release name'
 GENERATING_RELEASE_NOTES_MSG = 'Generating release notes since {previous_release}. \n' \
                                'You can see the progress here: \n' \
@@ -58,6 +66,8 @@ REL_NAME_REGEXP="(CMSSW_[0-9]+_[0-9]+)_[0-9]+(_SLHC[0-9]*|)(_pre[0-9]+|_[a-zA-Z]
 UPLOAD_COMMENT = 'upload %s'
 UPLOAD_ALL_COMMENT = '[uU]pload all'
 ABORT_COMMENT = '^[Aa]bort$'
+BUILD_TOOLCONF = '^[Bb]uild cmssw-tool-conf'
+APPROVAL_COMMENT =  '^[+]1$'
 DEFAULT_CHECK_COMMENT = ( REQUEST_BUILD_RELEASE + [ 'cmsbuild' ] )
 RELEASE_NOTES_GENERATED_LBL = 'release-notes-requested'
 JENKINS_PREV_RELEASE='PREVIOUS_RELEASE'
@@ -82,6 +92,8 @@ BUILD_IN_PROGRESS = 'build-in-progress'
 BUILD_STARTED = 'build-started'
 # The build has been aborted. 
 BUILD_ABORTED = 'build-aborted'
+# they requested to build cmssw-tool-conf and it is being built
+TOOLCONF_BUILDING = 'toolconf-building'
 
 # -------------------------------------------------------------------------------
 # Functions
@@ -186,8 +198,9 @@ def check_if_already_processed( issue ):
   return False
 #
 #  Creates the properties files to trigger the build in Jenkins
+# if only_toolconf is selected, it adds a parameter to tell the script to only build cmssw-tool-conf
 #
-def create_properties_files(release_name, architectures, issue_number, queue):
+def create_properties_files(release_name, architectures, issue_number, queue, only_toolconf=False):
   if opts.dryRun:
     print 'Not creating properties files for (dry-run): %s' % ", ".join( architectures )
     return
@@ -200,6 +213,8 @@ def create_properties_files(release_name, architectures, issue_number, queue):
     out_file.write( '%s=%s\n' % ( JENKINS_ARCH , arch ) )
     out_file.write( '%s=%s\n' % ( JENKINS_ISSUE_NUMBER , issue_number ) )
     out_file.write( '%s=%s\n' % ( JENKINS_CMSSW_QUEUE , queue) )
+    out_file.write( '%s=%s\n' % ( JENKINS_ONLY_TOOL_CONF, only_toolconf ) )
+
 
 #
 # generates the properties file for triggering the release notes
@@ -383,6 +398,8 @@ def get_issue_status( issue ):
     return PENDING_APPROVAL
   if BUILD_IN_PROGRESS in labels:
     return BUILD_IN_PROGRESS
+  if TOOLCONF_BUILDING in labels:
+    return TOOLCONF_BUILDING
 
 #
 # removes the labels of the issue
@@ -429,6 +446,10 @@ def fillDeatilsArchsLists( issue ):
   UPLOAD_OK.extend( [ x.split('-')[0] for x in labels if '-upload-ok' in x ] )
   UPLOADING.extend( [ x.split('-')[0] for x in labels if '-uploading' in x ] )
   BUILD_ERROR.extend( [ x.split('-')[0] for x in labels if '-build-error' in x ] )
+  TOOL_CONF_BUILDING.extend( [ x.split('-')[0] for x in labels if '-tool-conf-building' in x ] )
+  TOOL_CONF_OK.extend( [ x.split('-')[0] for x in labels if '-tool-conf-ok' in x ] )
+  TOOL_CONF_ERROR.extend( [ x.split('-')[0] for x in labels if '-tool-conf-error' in x ] )
+  TOOL_CONF_WAITING.extend( [ x.split('-')[0] for x in labels if '-tool-conf-waiting' in x ] )
   TO_CLEANUP.extend( UPLOAD_OK + BUILD_ERROR + TO_UPLOAD )
 
 #
@@ -456,6 +477,40 @@ def triggerCleanup( issue, comments, release_name ):
       remove_label( issue, arch + '-build-error' )
       add_label( issue, arch + '-finished' )
 
+#
+# Creates the release in github, including the cmssw tag. It then creates the files to trigger the builds in jenkins
+#TODO check cmssw toolconf
+#
+def start_release_build( issue, release_name, release_branch, architectures ): 
+
+  # if someone approved, go ahead and create the release
+  cmssw_repo = gh.get_organization( GH_CMSSW_ORGANIZATION ).get_repo( GH_CMSSW_REPO )
+  release_created = create_release_github( cmssw_repo, release_name, release_branch )
+  if not release_created:
+    msg = RELEASE_CREATION_FAIL_MSG.format( rel_name=release_name )
+    post_message( issue , RELEASE_CREATION_FAIL_MSG.format( rel_name=release_name ) )
+    exit( 0 )
+
+  msg = RELEASE_CREATED_MSG % ( RELEASE_BASE_URL % release_name )
+  post_message( issue , msg )
+
+  ready_to_build = list( set( architectures ) - set( TOOL_CONF_WAITING ) - set( TOOL_CONF_ERROR ) - set( TOOL_CONF_BUILDING ) )
+  create_properties_files( release_name, ready_to_build, issue.number, release_queue )
+  msg = QUEUING_BUILDS_MSG % ', '.join( architectures )
+  post_message( issue , msg )
+
+#
+# Creates the files to trigger the build of cmssw-tool-conf in jenkins.
+#
+def start_tool_conf_build( issue, release_name, release_branch, architectures ):
+
+  create_properties_files( release_name, architectures, issue.number, release_queue, only_toolconf=True )
+  QUEUING_TOOLCONF_MSG
+  msg = QUEUING_TOOLCONF_MSG % ', '.join( architectures )
+  post_message( issue , msg )
+  remove_labels( issue )
+  add_label( issue, TOOLCONF_BUILDING ) 
+
 
 # -------------------------------------------------------------------------------
 # Start of execution 
@@ -546,6 +601,10 @@ if __name__ == "__main__":
   UPLOADING = []
   BUILD_ERROR = []
   TO_CLEANUP = []
+  TOOL_CONF_BUILDING = []
+  TOOL_CONF_OK = []
+  TOOL_CONF_ERROR = []
+  TOOL_CONF_WAITING = []
   # These lists are filled by fillDeatilsArchsLists( issue )
 
   fillDeatilsArchsLists( issue )
@@ -579,28 +638,49 @@ if __name__ == "__main__":
 
   if status == PENDING_APPROVAL:
     approval_comments = search_in_comments( comments, APPROVE_BUILD_RELEASE , '^[+]1$', True )
-    if not approval_comments:
-      print 'Build not approved yet'
+    build_toolconf_commments = search_in_comments( comments, APPROVE_BUILD_RELEASE , BUILD_TOOLCONF, True )
+    is_patch = 'patch' in release_name
+
+    if build_toolconf_commments:
+      if is_patch:
+        post_message( issue, NOT_TOOLCONF_FOR_PATCH_MSG )
+      else:
+        start_tool_conf_build( issue, release_name, release_branch, architectures )
+    elif approval_comments:
+      start_release_build( issue, release_name, release_branch, architectures )
+      remove_labels( issue )
+      add_label( issue, BUILD_IN_PROGRESS )
+    else:
+      print 'Build not approved yet, or cmssw-tool-conf requested'
       exit( 0 )
 
-    # if someone approved, go ahead and create the release
-    cmssw_repo = gh.get_organization( GH_CMSSW_ORGANIZATION ).get_repo( GH_CMSSW_REPO )
-    release_created = create_release_github( cmssw_repo, release_name, release_branch )
-    if not release_created:
-      msg = RELEASE_CREATION_FAIL_MSG.format( rel_name=release_name )
-      post_message( issue , RELEASE_CREATION_FAIL_MSG.format( rel_name=release_name ) )
-      exit( 0 )
+  if status == TOOLCONF_BUILDING:
+    print 'Waiting for approval to start the build'
+    approval_comments = search_in_comments( comments, APPROVE_BUILD_RELEASE , APPROVAL_COMMENT, True )
+    if approval_comments:
+      print 'Build approved, switching to "Build in Progress" state'
+
+      # add a label for each arch for which tool conf has not started in jenkins
+      tool_conf_reported = ( TOOL_CONF_BUILDING + TOOL_CONF_OK + TOOL_CONF_ERROR )
+      not_started = list( set( architectures ) - set( tool_conf_reported ) )
 
-    msg = RELEASE_CREATED_MSG % ( RELEASE_BASE_URL % release_name )
-    post_message( issue , msg )
+      for arch in not_started:
+        add_label( issue, arch + '-tool-conf-waiting' )
+        TOOL_CONF_WAITING.append( arch )
 
-    create_properties_files( release_name, architectures, issue_id, release_queue )
-    msg = QUEUING_BUILDS_MSG % ', '.join( architectures )
-    post_message( issue , msg )
-    remove_labels( issue )
-    add_label( issue, BUILD_IN_PROGRESS )
+      remove_label( issue, TOOLCONF_BUILDING )
+      add_label( issue, BUILD_IN_PROGRESS ) 
+      start_release_build( issue, release_name, release_branch, architectures )
 
   if status == BUILD_IN_PROGRESS:
+
+    ready_to_build = list( set( architectures ) - set( TOOL_CONF_WAITING ) - set( TOOL_CONF_ERROR ) - set( TOOL_CONF_BUILDING ) )
+    print 'conf tool waiting'
+    print TOOL_CONF_WAITING
+    print 'ready to build'
+    print ready_to_build
+    create_properties_files( release_name, ready_to_build, issue.number, release_queue )
+
     # the build can be aborted only if none of the architectures has been uploaded or is uploading
     if not ( UPLOADING + UPLOAD_OK ):
       abort_comments = search_in_comments( comments , APPROVE_BUILD_RELEASE , ABORT_COMMENT, True )
@@ -671,6 +751,3 @@ if __name__ == "__main__":
       add_label( issue, arch + '-uploading' )
 
   exit( 0 )
-
-
-
diff --git a/report-build-release-status b/report-build-release-status
@@ -15,7 +15,10 @@ from os.path import expanduser
 GH_CMSSW_ORGANIZATION = 'cms-sw'
 GH_CMSSW_REPO = 'cmssw'
 POST_BUILDING='BUILDING'
+POST_TOOL_CONF_BUILDING='TOOL_CONF_BUILDING'
 BUILD_OK='BUILD_OK'
+TOOL_CONF_OK='TOOL_CONF_OK'
+TOOL_CONF_ERROR = 'TOOL_CONF_ERROR'
 BUILD_ERROR='BUILD_ERROR'
 UPLOADING='UPLOADING'
 UPLOAD_OK='UPLOAD_OK'
@@ -28,13 +31,23 @@ TESTS_OK='TESTS_OK'
 TESTS_ERROR='TESTS_ERROR'
 BUILDING_MSG='The build has started for {architecture} in {machine}. \n' \
              'You can see the progress here: https://cmssdt.cern.ch/jenkins/job/build-release/{jk_build_number}/console \n' \
-             '{details}' 
+             '{details}'
+BUILDING_TOOL_CONF_MSG='The cmssw-tool-conf build has started for {architecture} in {machine}. \n' \
+             'You can see the progress here: https://cmssdt.cern.ch/jenkins/job/build-release/{jk_build_number}/console \n' \
+             '{details}'
 BUILD_OK_MSG='The build has finished sucessfully for the architecture {architecture} and is ready to be uploaded. \n' \
              '* You can start the uploads by writing the comment: "upload all". I will upload all the architectures as soon as the build finishes successfully.\n' \
              '* Or you can start the upload only for {architecture} by writing: "upload {architecture}".\n \n' \
              'You can see the log for the build here: \n' \
              '{log_url} \n' \
              'Some tests ( runTheMatrix.py -s ) are being run, the results will be posted when done.'
+TOOL_CONF_OK_MSG='The cmssw-tool-conf build has finished sucessfully for the architecture {architecture} and it was automatically uploaded. \n' \
+             'Remember that if you write "+1" I will start to build this and all the architectures as soon as their cmssw-tool-conf finish.\n' \
+             'You can see the log for the build here: \n' \
+             '{log_url} \n' 
+TOOL_CONF_ERROR_MSG='There was an error building cmssw-tool-conf for {architecture} \n' \
+             'You can see the log for the build here: \n' \
+             '{log_url} \n'
 UPLOADING_MSG='The upload has started for {architecture} in {machine}. \n' \
               'You can see the progress here: https://cmssdt.cern.ch/jenkins/job/upload-release/{jk_build_number}/console'
 UPLOAD_OK_MSG='The upload has successfully finished for {architecture} \n You can see the log here: \n {log_url} \n' \
@@ -51,7 +64,7 @@ TESTS_ERROR_MSG='There was an error when running the tests for {architecture} \n
 BUILD_ERROR_MSG='The was an error for {architecture}. \n You can see the log here: \n {log_url}'
 BUILD_QUEUED_LABEL = 'build-release-queued'
 BUILD_STARTED = 'build-release-started'
-BASE_BUILD_LOG_URL = 'https://cmssdt.cern.ch/SDT/jenkins-artifacts/auto-build-release/%s-%s/%d' 
+BASE_BUILD_LOG_URL = 'https://cmssdt.cern.ch/SDT/jenkins-artifacts/auto-build-release/%s-%s/%d'
 BASE_UPLOAD_LOG_URL = 'https://cmssdt.cern.ch/SDT/jenkins-artifacts/auto-upload-release/%s-%s/%d'
 BASE_CLEANUP_LOG_URL = 'https://cmssdt.cern.ch/SDT/jenkins-artifacts/cleanup-auto-build/%s-%s/%d'
 
@@ -131,6 +144,17 @@ if __name__ == "__main__":
     new_label = arch+'-building'
     add_label( issue, new_label )
 
+  elif action == POST_TOOL_CONF_BUILDING:
+
+    msg_details = ''
+    if opts.details:
+      msg_details = opts.details
+    msg = BUILDING_TOOL_CONF_MSG.format( architecture=arch, machine=hostname, jk_build_number=jenkins_build_number, details=msg_details )
+    post_message( issue , msg )
+    remove_label( issue, arch+'-tool-conf-waiting' )
+    new_label = arch+'-tool-conf-building'
+    add_label( issue, new_label )
+
   elif action == BUILD_OK:
 
     results_url = BASE_BUILD_LOG_URL % (release_name,arch,jenkins_build_number)
@@ -139,6 +163,14 @@ if __name__ == "__main__":
     remove_label( issue, arch+'-building' )
     add_label( issue,  arch+'-build-ok' )
 
+  elif action == TOOL_CONF_OK:
+
+    results_url = BASE_BUILD_LOG_URL % (release_name,arch,jenkins_build_number)
+    msg = TOOL_CONF_OK_MSG.format( architecture=arch, log_url=results_url )
+    post_message( issue , msg )
+    remove_label( issue, arch+'-tool-conf-building' )
+    add_label( issue,  arch+'-tool-conf-ok' )
+
   elif action == BUILD_ERROR:
 
     results_url = BASE_BUILD_LOG_URL % (release_name,arch,jenkins_build_number)
@@ -147,6 +179,14 @@ if __name__ == "__main__":
     remove_label( issue, arch+'-building' )
     add_label( issue,  arch+'-build-error' )
 
+  elif action == TOOL_CONF_ERROR:
+
+    results_url = BASE_BUILD_LOG_URL % (release_name,arch,jenkins_build_number)
+    msg = TOOL_CONF_ERROR_MSG.format( architecture=arch, log_url=results_url )
+    post_message( issue , msg )
+    remove_label( issue, arch+'-tool-conf-building' )
+    add_label( issue,  arch+'-tool-conf-error' )
+
   elif action == UPLOADING:
 
     msg = UPLOADING_MSG.format( architecture=arch, machine=hostname, jk_build_number=jenkins_build_number)