From 0abd3510eab4302767843e5862adf771a5d9e921 Mon Sep 17 00:00:00 2001 From: Danny McCormick Date: Fri, 28 Apr 2023 11:15:30 -0400 Subject: [PATCH] keep retrying mass_comment until it has started all jobs (#26457) * keep retrying mass_comment until it has started all jobs * fix lookups --- release/src/main/scripts/mass_comment.py | 368 +++++++++++++---------- 1 file changed, 210 insertions(+), 158 deletions(-) diff --git a/release/src/main/scripts/mass_comment.py b/release/src/main/scripts/mass_comment.py index 99c1ac646d6e..48582edc5a65 100644 --- a/release/src/main/scripts/mass_comment.py +++ b/release/src/main/scripts/mass_comment.py @@ -23,154 +23,152 @@ # This list can be found by querying the Jenkins API, see BEAM-13951 COMMENTS_TO_ADD = [ - "Run CommunityMetrics PreCommit", - "Run Dataflow Runner Nexmark Tests", - "Run Dataflow Runner Tpcds Tests", - "Run Dataflow Runner V2 Java 11 Nexmark Tests", - "Run Dataflow Runner V2 Java 17 Nexmark Tests", - "Run Dataflow Runner V2 Nexmark Tests", - "Run Dataflow Streaming ValidatesRunner", - "Run Dataflow ValidatesRunner Java 11", - "Run Dataflow ValidatesRunner Java 17", - "Run Dataflow ValidatesRunner", - "Run Direct Runner Nexmark Tests", - "Run Direct ValidatesRunner Java 11", - "Run Direct ValidatesRunner Java 17", - "Run Direct ValidatesRunner in Java 11", - "Run Direct ValidatesRunner", - "Run Flink Runner Nexmark Tests", - "Run Flink ValidatesRunner Java 11", - "Run Flink Runner Tpcds Tests", - "Run Flink ValidatesRunner", - "Run Go Flink ValidatesRunner", - "Run Go PostCommit", - "Run Go PreCommit", - "Run Go Samza ValidatesRunner", - "Run Go Spark ValidatesRunner", - "Run GoPortable PreCommit", - "Run Java 11 Examples on Dataflow Runner V2", - "Run Java 17 Examples on Dataflow Runner V2", - "Run Java Dataflow V2 ValidatesRunner Streaming", - "Run Java Dataflow V2 ValidatesRunner", - "Run Java Examples on Dataflow Runner V2", - "Run Java Examples_Direct", - "Run Java Examples_Flink", - "Run Java Examples_Spark", - "Run Java Flink PortableValidatesRunner Streaming", - "Run Java PostCommit", - "Run Java PreCommit", - "Run Java Samza PortableValidatesRunner", - "Run Java Spark PortableValidatesRunner Batch", - "Run Java Spark v2 PortableValidatesRunner Streaming", - "Run Java Spark v3 PortableValidatesRunner Streaming", - "Run Java examples on Dataflow Java 11", - "Run Java examples on Dataflow Java 17", - "Run Java examples on Dataflow with Java 11", - "Run Java_Amazon-Web-Services2_IO_Direct PreCommit", - "Run Java_Amazon-Web-Services_IO_Direct PreCommit", - "Run Java_Amqp_IO_Direct PreCommit", - "Run Java_Azure_IO_Direct PreCommit", - "Run Java_Cassandra_IO_Direct PreCommit", - "Run Java_Cdap_IO_Direct PreCommit", - "Run Java_Clickhouse_IO_Direct PreCommit", - "Run Java_Debezium_IO_Direct PreCommit", - "Run Java_ElasticSearch_IO_Direct PreCommit", - "Run Java_Examples_Dataflow PreCommit", - "Run Java_Examples_Dataflow_Java11 PreCommit", - "Run Java_Examples_Dataflow_Java17 PreCommit", - "Run Java_GCP_IO_Direct PreCommit", - "Run Java_HBase_IO_Direct PreCommit", - "Run Java_HCatalog_IO_Direct PreCommit", - "Run Java_Hadoop_IO_Direct PreCommit", - "Run Java_InfluxDb_IO_Direct PreCommit", - "Run Java_JDBC_IO_Direct PreCommit", - "Run Java_Jms_IO_Direct PreCommit", - "Run Java_Kafka_IO_Direct PreCommit", - "Run Java_Kinesis_IO_Direct PreCommit", - "Run Java_Kudu_IO_Direct PreCommit", - "Run Java_MongoDb_IO_Direct PreCommit", - "Run Java_Mqtt_IO_Direct PreCommit", - "Run Java_Neo4j_IO_Direct PreCommit", - "Run Java_PVR_Flink_Batch PreCommit", - "Run Java_PVR_Flink_Docker PreCommit", - "Run Java_Parquet_IO_Direct PreCommit", - "Run Java_Pulsar_IO_Direct PreCommit", - "Run Java_RabbitMq_IO_Direct PreCommit", - "Run Java_Redis_IO_Direct PreCommit", - "Run Java_SingleStore_IO_Direct PreCommit", - "Run Java_Snowflake_IO_Direct PreCommit", - "Run Java_Solr_IO_Direct PreCommit", - "Run Java_Spark3_Versions PreCommit", - "Run Java_Splunk_IO_Direct PreCommit", - "Run Java_Thrift_IO_Direct PreCommit", - "Run Java_Tika_IO_Direct PreCommit", - "Run Javadoc PostCommit", - "Run Jpms Dataflow Java 11 PostCommit", - "Run Jpms Dataflow Java 17 PostCommit", - "Run Jpms Direct Java 11 PostCommit", - "Run Jpms Direct Java 17 PostCommit", - "Run Jpms Flink Java 11 PostCommit", - "Run Jpms Spark Java 11 PostCommit", - "Run Kotlin_Examples PreCommit", - "Run PortableJar_Flink PostCommit", - "Run PortableJar_Spark PostCommit", - "Run Portable_Python PreCommit", - "Run PostCommit_Java_Dataflow", - "Run PostCommit_Java_DataflowV2", - "Run PostCommit_Java_Hadoop_Versions", - "Run Python 3.7 PostCommit", - "Run Python 3.8 PostCommit", - "Run Python 3.9 PostCommit", - "Run Python 3.10 PostCommit", - "Run Python Dataflow V2 ValidatesRunner", - "Run Python Dataflow ValidatesContainer", - "Run Python Dataflow ValidatesRunner", - "Run Python Direct Runner Nexmark Tests", - "Run Python Examples_Dataflow", - "Run Python Examples_Direct", - "Run Python Examples_Flink", - "Run Python Examples_Spark", - "Run Python Flink ValidatesRunner", - "Run Python PreCommit", - "Run Python Samza ValidatesRunner", - "Run Python Spark ValidatesRunner", - "Run PythonDocker PreCommit", - "Run PythonDocs PreCommit", - "Run PythonFormatter PreCommit", - "Run PythonLint PreCommit", - "Run Python_Coverage PreCommit", - "Run Python_Dataframes PreCommit", - "Run Python_Examples PreCommit", - "Run Python_Integration PreCommit", - "Run Python_PVR_Flink PreCommit", - "Run Python_Runners PreCommit", - "Run Python_Transforms PreCommit", - "Run RAT PreCommit", - "Run Release Gradle Build", - "Run SQL PostCommit", - "Run SQL PreCommit", - "Run SQL_Java11 PreCommit", - "Run SQL_Java17 PreCommit", - "Run Samza ValidatesRunner", - "Run Spark Runner Nexmark Tests", - "Run Spark Runner Tpcds Tests", - "Run Spark StructuredStreaming ValidatesRunner", - "Run Spark ValidatesRunner Java 11", - "Run Spark ValidatesRunner", - "Run Spotless PreCommit", - "Run Twister2 ValidatesRunner", - "Run Typescript PreCommit", - "Run ULR Loopback ValidatesRunner", - "Run Whitespace PreCommit", - "Run XVR_Direct PostCommit", - "Run XVR_Flink PostCommit", - "Run XVR_GoUsingJava_Dataflow PostCommit", - "Run XVR_JavaUsingPython_Dataflow PostCommit", - "Run XVR_PythonUsingJavaSQL_Dataflow PostCommit", - "Run XVR_PythonUsingJava_Dataflow PostCommit", - "Run XVR_Samza PostCommit", - "Run XVR_Spark PostCommit", - "Run XVR_Spark3 PostCommit", + ("Run CommunityMetrics PreCommit", "beam_PreCommit_CommunityMetrics"), + ("Run Dataflow Runner Nexmark Tests", "beam_PostCommit_Java_Nexmark_Dataflow"), + ("Run Dataflow Runner Tpcds Tests", "beam_PostCommit_Java_Tpcds_Dataflow"), + ("Run Dataflow Runner V2 Java 11 Nexmark Tests", "beam_PostCommit_Java_Nexmark_DataflowV2_Java11"), + ("Run Dataflow Runner V2 Java 17 Nexmark Tests", "beam_PostCommit_Java_Nexmark_DataflowV2_Java17"), + ("Run Dataflow Runner V2 Nexmark Tests", "beam_PostCommit_Java_Nexmark_DataflowV2"), + ("Run Dataflow Streaming ValidatesRunner", "beam_PostCommit_Java_ValidatesRunner_Dataflow_Streaming"), + ("Run Dataflow ValidatesRunner Java 11", "beam_PostCommit_Java_ValidatesRunner_Dataflow_Java11"), + ("Run Dataflow ValidatesRunner Java 17", "beam_PostCommit_Java_ValidatesRunner_Dataflow_Java17"), + ("Run Dataflow ValidatesRunner", "beam_PostCommit_Java_ValidatesRunner_Dataflow"), + ("Run Direct Runner Nexmark Tests", "beam_PostCommit_Java_Nexmark_Direct"), + ("Run Direct ValidatesRunner Java 11", "beam_PostCommit_Java_ValidatesRunner_Direct_Java11"), + ("Run Direct ValidatesRunner Java 17", "beam_PostCommit_Java_ValidatesRunner_Direct_Java17"), + ("Run Direct ValidatesRunner Java 11", "beam_PostCommit_Java_ValidatesRunner_Direct_Java11"), + ("Run Direct ValidatesRunner", "beam_PostCommit_Java_ValidatesRunner_Direct"), + ("Run Flink Runner Nexmark Tests", "beam_PostCommit_Java_Nexmark_Flink"), + ("Run Flink ValidatesRunner Java 11", "beam_PostCommit_Java_ValidatesRunner_Flink_Java11"), + ("Run Flink Runner Tpcds Tests", "beam_PostCommit_Java_Tpcds_Flink"), + ("Run Flink ValidatesRunner", "beam_PostCommit_Java_ValidatesRunner_Flink"), + ("Run Go Flink ValidatesRunner", "beam_PostCommit_Go_VR_Flink"), + ("Run Go PostCommit", "beam_PostCommit_Go"), + ("Run Go PreCommit", "beam_PreCommit_Go"), + ("Run Go Samza ValidatesRunner", "beam_PostCommit_Go_VR_Samza"), + ("Run Go Spark ValidatesRunner", "beam_PostCommit_Go_VR_Spark"), + ("Run GoPortable PreCommit", "beam_PreCommit_GoPortable"), + ("Run Java 11 Examples on Dataflow Runner V2", "beam_PostCommit_Java_Examples_Dataflow_V2_java11"), + ("Run Java 17 Examples on Dataflow Runner V2", "beam_PostCommit_Java_Examples_Dataflow_V2_java17"), + ("Run Java Dataflow V2 ValidatesRunner Streaming", "beam_PostCommit_Java_VR_Dataflow_V2_Streaming"), + ("Run Java Dataflow V2 ValidatesRunner", "beam_PostCommit_Java_VR_Dataflow_V2"), + ("Run Java Examples on Dataflow Runner V2", "beam_PostCommit_Java_Examples_Dataflow_V2"), + ("Run Java Examples_Direct", "beam_PostCommit_Java_Examples_Direct"), + ("Run Java Examples_Flink", "beam_PostCommit_Java_Examples_Flink"), + ("Run Java Examples_Spark", "beam_PostCommit_Java_Examples_Spark"), + ("Run Java Flink PortableValidatesRunner Streaming", "beam_PostCommit_Java_PVR_Flink_Streaming"), + ("Run Java PostCommit", "beam_PostCommit_Java"), + ("Run Java PreCommit", "beam_PreCommit_Java"), + ("Run Java Samza PortableValidatesRunner", "beam_PostCommit_Java_PVR_Samza"), + ("Run Java Spark PortableValidatesRunner Batch", "beam_PostCommit_Java_PVR_Spark_Batch"), + ("Run Java Spark v3 PortableValidatesRunner Streaming", "beam_PostCommit_Java_PVR_Spark3_Streaming"), + ("Run Java examples on Dataflow Java 11", "beam_PostCommit_Java_Examples_Dataflow_Java11"), + ("Run Java examples on Dataflow Java 17", "beam_PostCommit_Java_Examples_Dataflow_Java17"), + ("Run Java_Amazon-Web-Services2_IO_Direct PreCommit", "beam_PreCommit_Java_Amazon-Web-Services2_IO_Direct"), + ("Run Java_Amazon-Web-Services_IO_Direct PreCommit", "beam_PreCommit_Java_Amazon-Web-Services_IO_Direct"), + ("Run Java_Amqp_IO_Direct PreCommit", "beam_PreCommit_Java_Amqp_IO_Direct"), + ("Run Java_Azure_IO_Direct PreCommit", "beam_PreCommit_Java_Azure_IO_Direct"), + ("Run Java_Cassandra_IO_Direct PreCommit", "beam_PreCommit_Java_Cassandra_IO_Direct"), + ("Run Java_Cdap_IO_Direct PreCommit", "beam_PreCommit_Java_Cdap_IO_Direct"), + ("Run Java_Clickhouse_IO_Direct PreCommit", "beam_PreCommit_Java_Clickhouse_IO_Direct"), + ("Run Java_Debezium_IO_Direct PreCommit", "beam_PreCommit_Java_Debezium_IO_Direct"), + ("Run Java_ElasticSearch_IO_Direct PreCommit", "beam_PreCommit_Java_ElasticSearch_IO_Direct"), + ("Run Java_Examples_Dataflow PreCommit", "beam_PreCommit_Java_Examples_Dataflow"), + ("Run Java_Examples_Dataflow_Java11 PreCommit", "beam_PreCommit_Java_Examples_Dataflow_Java11"), + ("Run Java_Examples_Dataflow_Java17 PreCommit", "beam_PreCommit_Java_Examples_Dataflow_Java17"), + ("Run Java_GCP_IO_Direct PreCommit", "beam_PreCommit_Java_GCP_IO_Direct"), + ("Run Java_HBase_IO_Direct PreCommit", "beam_PreCommit_Java_HBase_IO_Direct"), + ("Run Java_HCatalog_IO_Direct PreCommit", "beam_PreCommit_Java_HCatalog_IO_Direct"), + ("Run Java_Hadoop_IO_Direct PreCommit", "beam_PreCommit_Java_Hadoop_IO_Direct"), + ("Run Java_InfluxDb_IO_Direct PreCommit", "beam_PreCommit_Java_InfluxDb_IO_Direct"), + ("Run Java_JDBC_IO_Direct PreCommit", "beam_PreCommit_Java_JDBC_IO_Direct"), + ("Run Java_Jms_IO_Direct PreCommit", "beam_PreCommit_Java_Jms_IO_Direct"), + ("Run Java_Kafka_IO_Direct PreCommit", "beam_PreCommit_Java_Kafka_IO_Direct"), + ("Run Java_Kinesis_IO_Direct PreCommit", "beam_PreCommit_Java_Kinesis_IO_Direct"), + ("Run Java_Kudu_IO_Direct PreCommit", "beam_PreCommit_Java_Kudu_IO_Direct"), + ("Run Java_MongoDb_IO_Direct PreCommit", "beam_PreCommit_Java_MongoDb_IO_Direct"), + ("Run Java_Mqtt_IO_Direct PreCommit", "beam_PreCommit_Java_Mqtt_IO_Direct"), + ("Run Java_Neo4j_IO_Direct PreCommit", "beam_PreCommit_Java_Neo4j_IO_Direct"), + ("Run Java_PVR_Flink_Batch PreCommit", "beam_PreCommit_Java_PVR_Flink_Batch"), + ("Run Java_PVR_Flink_Docker PreCommit", "beam_PreCommit_Java_PVR_Flink_Docker"), + ("Run Java_Parquet_IO_Direct PreCommit", "beam_PreCommit_Java_Parquet_IO_Direct"), + ("Run Java_Pulsar_IO_Direct PreCommit", "beam_PreCommit_Java_Pulsar_IO_Direct"), + ("Run Java_RabbitMq_IO_Direct PreCommit", "beam_PreCommit_Java_RabbitMq_IO_Direct"), + ("Run Java_Redis_IO_Direct PreCommit", "beam_PreCommit_Java_Redis_IO_Direct"), + ("Run Java_SingleStore_IO_Direct PreCommit", "beam_PreCommit_Java_SingleStore_IO_Direct"), + ("Run Java_Snowflake_IO_Direct PreCommit", "beam_PreCommit_Java_Snowflake_IO_Direct"), + ("Run Java_Solr_IO_Direct PreCommit", "beam_PreCommit_Java_Solr_IO_Direct"), + ("Run Java_Spark3_Versions PreCommit", "beam_PreCommit_Java_Spark3_Versions"), + ("Run Java_Splunk_IO_Direct PreCommit", "beam_PreCommit_Java_Splunk_IO_Direct"), + ("Run Java_Thrift_IO_Direct PreCommit", "beam_PreCommit_Java_Thrift_IO_Direct"), + ("Run Java_Tika_IO_Direct PreCommit", "beam_PreCommit_Java_Tika_IO_Direct"), + ("Run Javadoc PostCommit", "beam_PostCommit_Javadoc"), + ("Run Jpms Dataflow Java 11 PostCommit", "beam_PostCommit_Java_Jpms_Dataflow_Java11"), + ("Run Jpms Dataflow Java 17 PostCommit", "beam_PostCommit_Java_Jpms_Dataflow_Java17"), + ("Run Jpms Direct Java 11 PostCommit", "beam_PostCommit_Java_Jpms_Direct_Java11"), + ("Run Jpms Direct Java 17 PostCommit", "beam_PostCommit_Java_Jpms_Direct_Java17"), + ("Run Jpms Flink Java 11 PostCommit", "beam_PostCommit_Java_Jpms_Flink_Java11"), + ("Run Jpms Spark Java 11 PostCommit", "beam_PostCommit_Java_Jpms_Spark_Java11"), + ("Run Kotlin_Examples PreCommit", "beam_PreCommit_Kotlin_Examples"), + ("Run PortableJar_Flink PostCommit", "beam_PostCommit_PortableJar_Flink"), + ("Run PortableJar_Spark PostCommit", "beam_PostCommit_PortableJar_Spark"), + ("Run Portable_Python PreCommit", "beam_PreCommit_Portable_Python"), + ("Run PostCommit_Java_Dataflow", "beam_PostCommit_Java_DataflowV1"), + ("Run PostCommit_Java_DataflowV2", "beam_PostCommit_Java_DataflowV2"), + ("Run PostCommit_Java_Hadoop_Versions", "beam_PostCommit_Java_Hadoop_Versions"), + ("Run Python 3.7 PostCommit", "beam_PostCommit_Python37"), + ("Run Python 3.8 PostCommit", "beam_PostCommit_Python38"), + ("Run Python 3.9 PostCommit", "beam_PostCommit_Python39"), + ("Run Python 3.10 PostCommit", "beam_PostCommit_Python310"), + ("Run Python 3.11 PostCommit", "beam_PostCommit_Python311"), + ("Run Python Dataflow V2 ValidatesRunner", "beam_PostCommit_Py_VR_Dataflow_V2"), + ("Run Python Dataflow ValidatesContainer", "beam_PostCommit_Py_ValCont"), + ("Run Python Dataflow ValidatesRunner", "beam_PostCommit_Py_VR_Dataflow"), + ("Run Python Direct Runner Nexmark Tests", "beam_PostCommit_Python_Nexmark_Direct"), + ("Run Python Examples_Dataflow", "beam_PostCommit_Python_Examples_Dataflow"), + ("Run Python Examples_Direct", "beam_PostCommit_Python_Examples_Direct"), + ("Run Python Examples_Flink", "beam_PostCommit_Python_Examples_Flink"), + ("Run Python Examples_Spark", "beam_PostCommit_Python_Examples_Spark"), + ("Run Python Flink ValidatesRunner", "beam_PostCommit_Python_VR_Flink"), + ("Run Python PreCommit", "beam_PreCommit_Python"), + ("Run Python Samza ValidatesRunner", "beam_PostCommit_Python_VR_Samza"), + ("Run Python Spark ValidatesRunner", "beam_PostCommit_Python_VR_Spark"), + ("Run PythonDocker PreCommit", "beam_PreCommit_PythonDocker"), + ("Run PythonDocs PreCommit", "beam_PreCommit_PythonDocs"), + ("Run PythonFormatter PreCommit", "beam_PreCommit_PythonFormatter"), + ("Run PythonLint PreCommit", "beam_PreCommit_PythonLint"), + ("Run Python_Coverage PreCommit", "beam_PreCommit_Python_Coverage"), + ("Run Python_Dataframes PreCommit", "beam_PreCommit_Python_Dataframes"), + ("Run Python_Examples PreCommit", "beam_PreCommit_Python_Examples"), + ("Run Python_Integration PreCommit", "beam_PreCommit_Python_Integration"), + ("Run Python_PVR_Flink PreCommit", "beam_PreCommit_Python_PVR_Flink"), + ("Run Python_Runners PreCommit", "beam_PreCommit_Python_Runners"), + ("Run Python_Transforms PreCommit", "beam_PreCommit_Python_Transforms"), + ("Run RAT PreCommit", "beam_PreCommit_RAT"), + ("Run Release Gradle Build", "beam_Release_Gradle_Build"), + ("Run SQL PostCommit", "beam_PostCommit_SQL"), + ("Run SQL PreCommit", "beam_PreCommit_SQL"), + ("Run SQL_Java11 PreCommit", "beam_PreCommit_SQL_Java11"), + ("Run SQL_Java17 PreCommit", "beam_PreCommit_SQL_Java17"), + ("Run Samza ValidatesRunner", "beam_PostCommit_Java_ValidatesRunner_Samza"), + ("Run Spark Runner Nexmark Tests", "beam_PostCommit_Java_Nexmark_Spark"), + ("Run Spark Runner Tpcds Tests", "beam_PostCommit_Java_Tpcds_Spark"), + ("Run Spark StructuredStreaming ValidatesRunner", "beam_PostCommit_Java_ValidatesRunner_SparkStructuredStreaming"), + ("Run Spark ValidatesRunner Java 11", "beam_PostCommit_Java_ValidatesRunner_Spark_Java11"), + ("Run Spark ValidatesRunner", "beam_PostCommit_Java_ValidatesRunner_Spark"), + ("Run Spotless PreCommit", "beam_PreCommit_Spotless"), + ("Run Twister2 ValidatesRunner", "beam_PostCommit_Java_ValidatesRunner_Twister2"), + ("Run Typescript PreCommit", "beam_PreCommit_Typescript"), + ("Run ULR Loopback ValidatesRunner", "beam_PostCommit_Java_ValidatesRunner_ULR"), + ("Run Whitespace PreCommit", "beam_PreCommit_Whitespace"), + ("Run XVR_Direct PostCommit", "beam_PostCommit_XVR_Direct"), + ("Run XVR_Flink PostCommit", "beam_PostCommit_XVR_Flink"), + ("Run XVR_GoUsingJava_Dataflow PostCommit", "beam_PostCommit_XVR_GoUsingJava_Dataflow"), + ("Run XVR_JavaUsingPython_Dataflow PostCommit", "beam_PostCommit_XVR_JavaUsingPython_Dataflow"), + ("Run XVR_PythonUsingJavaSQL_Dataflow PostCommit", "beam_PostCommit_XVR_PythonUsingJavaSQL_Dataflow"), + ("Run XVR_PythonUsingJava_Dataflow PostCommit", "beam_PostCommit_XVR_PythonUsingJava_Dataflow"), + ("Run XVR_Samza PostCommit", "beam_PostCommit_XVR_Samza"), + ("Run XVR_Spark3 PostCommit", "beam_PostCommit_XVR_Spark3"), ] @@ -196,8 +194,8 @@ def getSubjectId(accessToken, prNumber): return response['data']['repository']['pullRequest']['id'] -def fetchGHData(accessToken, subjectId, commentBody): - '''Fetches GitHub data required for reporting Beam metrics''' +def addPrComment(accessToken, subjectId, commentBody): + '''Adds a pr comment to the PR defined by subjectId''' query = ''' mutation AddPullRequestComment { addComment(input:{subjectId:"%s",body: "%s"}) { @@ -215,17 +213,40 @@ def fetchGHData(accessToken, subjectId, commentBody): ''' % (subjectId, commentBody) return executeGHGraphqlQuery(accessToken, query) +def getPrStatuses(accessToken, prNumber): + query = ''' +query GetPRChecks { + repository(name: "beam", owner: "apache") { + pullRequest(number: %s) { + commits(last: 1) { + nodes { + commit { + status { + contexts { + targetUrl + context + } + } + } + } + } + } + } +} +''' % (prNumber) + return executeGHGraphqlQuery(accessToken, query) + -def postComments(accessToken, subjectId): +def postComments(accessToken, subjectId, commentsToAdd): ''' - Main workhorse method. Fetches data from GitHub and puts it in metrics table. + Main workhorse method. Posts comments to GH. ''' - for commentBody in COMMENTS_TO_ADD: - jsonData = fetchGHData(accessToken, subjectId, commentBody) + for comment in commentsToAdd: + jsonData = addPrComment(accessToken, subjectId, comment[0]) + print(jsonData) # Space out comments 30 seconds apart to avoid overwhelming Jenkins time.sleep(30) - print(jsonData) def probeGitHubIsUp(): @@ -236,6 +257,21 @@ def probeGitHubIsUp(): result = sock.connect_ex(('github.com', 443)) return True if result == 0 else False +def getRemainingComments(accessToken, pr, initialComments): + ''' + Filters out the comments that already have statuses associated with them from initial comments + ''' + queryResult = getPrStatuses(accessToken, pr) + pull = queryResult["data"]["repository"]["pullRequest"] + commit = pull["commits"]["nodes"][0]["commit"] + check_urls = str(list(map(lambda c : c["targetUrl"], commit["status"]["contexts"]))) + remainingComments = [] + for comment in initialComments: + if f'/{comment[1]}_Phrase/' not in check_urls and f'/{comment[1]}_PR/' not in check_urls and f'/{comment[1]}_Commit/' not in check_urls and f'/{comment[1]}/' not in check_urls: + print(comment) + remainingComments.append(comment) + return remainingComments + ################################################################################ if __name__ == '__main__': @@ -256,8 +292,24 @@ def probeGitHubIsUp(): pr = input("Enter the Beam PR number to test (e.g. 11403): ") subjectId = getSubjectId(accessToken, pr) - - postComments(accessToken, subjectId) - print("Fetched data.") + + remainingComments = getRemainingComments(accessToken, pr, COMMENTS_TO_ADD) + if len(remainingComments) == 0: + print('Jobs have been started for all comments. If you would like to retry all jobs, create a new commit before running this script.') + while len(remainingComments) > 0: + postComments(accessToken, subjectId, remainingComments) + # Sleep 60 seconds to allow checks to start to status + time.sleep(60) + remainingComments = getRemainingComments(accessToken, pr, remainingComments) + if len(remainingComments) > 0: + print(f'{len(remainingComments)} comments must be reposted because no check has been created for them: {str(remainingComments)}') + print('Sleeping for 1 hour to allow Jenkins to recover and to give it time to status.') + for i in range(60): + time.sleep(60) + print(f'{i} minutes elapsed, {60-i} minutes remaining') + remainingComments = getRemainingComments(accessToken, pr, remainingComments) + if len(remainingComments) == 0: + print(f'{len(remainingComments)} comments still must be reposted: {str(remainingComments)}') + print('Trying to repost comments.') print('Done.')