From 3d8db78984e80f546ce43128654c895a9524fb93 Mon Sep 17 00:00:00 2001 From: Yi Hu Date: Mon, 11 Jul 2022 12:48:51 -0400 Subject: [PATCH] Split checkStyle from precommit into spotless job (#22203) --- ...Commit_Java_Examples_Dataflow_V2_Java11.groovy | 1 + ...Commit_Java_Examples_Dataflow_V2_Java17.groovy | 1 + .test-infra/jenkins/job_PreCommit_Java.groovy | 6 ++---- .../job_PreCommit_Java_AWS2_IO_Direct.groovy | 3 ++- .../job_PreCommit_Java_AWS_IO_Direct.groovy | 3 ++- .../job_PreCommit_Java_Debezium_IO_Direct.groovy | 3 ++- .../job_PreCommit_Java_Examples_Dataflow.groovy | 3 ++- ...PreCommit_Java_Examples_Dataflow_Java11.groovy | 2 ++ .../job_PreCommit_Java_GCP_IO_Direct.groovy | 3 ++- .../job_PreCommit_Java_JDBC_IO_Direct.groovy | 3 ++- .../job_PreCommit_Java_Kafka_IO_Direct.groovy | 3 ++- .../job_PreCommit_Java_Kinesis_IO_Direct.groovy | 3 ++- .../job_PreCommit_Java_Neo4j_IO_Direct.groovy | 3 ++- .test-infra/jenkins/job_PreCommit_SQL.groovy | 7 ++----- .../jenkins/job_PreCommit_SQL_Java11.groovy | 4 +--- .../jenkins/job_PreCommit_SQL_Java17.groovy | 4 +--- .test-infra/jenkins/job_PreCommit_Spotless.groovy | 15 +++++++++++++-- ...Precommit_Java_Examples_Dataflow_Java17.groovy | 2 ++ .test-infra/jenkins/job_sonarqube_report.groovy | 2 +- .../apache/beam/gradle/BeamModulePlugin.groovy | 6 ++++++ 20 files changed, 50 insertions(+), 27 deletions(-) diff --git a/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java11.groovy b/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java11.groovy index 5b85e63fb701..6687ae0e6f8a 100644 --- a/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java11.groovy +++ b/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java11.groovy @@ -42,6 +42,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_Examples_Dataflow_V2_ja // overloading the machines. commonJobProperties.setGradleSwitches(delegate, 3 * Runtime.runtime.availableProcessors()) switches '-PdisableSpotlessCheck=true' + switches '-PdisableCheckStyle=true' switches '-PcompileAndRunTestsWithJava11' switches '-PskipCheckerFramework' switches "-Pjava11Home=${commonJobProperties.JAVA_11_HOME}" diff --git a/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java17.groovy b/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java17.groovy index 7e507aaf9e24..b275fe9276d9 100644 --- a/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java17.groovy +++ b/.test-infra/jenkins/job_PostCommit_Java_Examples_Dataflow_V2_Java17.groovy @@ -42,6 +42,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java_Examples_Dataflow_V2_ja // overloading the machines. commonJobProperties.setGradleSwitches(delegate, 3 * Runtime.runtime.availableProcessors()) switches '-PdisableSpotlessCheck=true' + switches '-PdisableCheckStyle=true' switches '-PcompileAndRunTestsWithJava17' switches '-PskipCheckerFramework' switches "-Pjava17Home=${commonJobProperties.JAVA_17_HOME}" diff --git a/.test-infra/jenkins/job_PreCommit_Java.groovy b/.test-infra/jenkins/job_PreCommit_Java.groovy index 0b4be9d633bb..f5e593bcf4b9 100644 --- a/.test-infra/jenkins/job_PreCommit_Java.groovy +++ b/.test-infra/jenkins/job_PreCommit_Java.groovy @@ -23,7 +23,8 @@ PrecommitJobBuilder builder = new PrecommitJobBuilder( nameBase: 'Java', gradleTask: ':javaPreCommit', gradleSwitches: [ - '-PdisableSpotlessCheck=true' + '-PdisableSpotlessCheck=true', + '-PdisableCheckStyle=true' ], // spotless checked in separate pre-commit timeoutMins: 180, triggerPathPatterns: [ @@ -45,9 +46,6 @@ builder.build { tools { errorProne() java() - checkStyle { - pattern('**/build/reports/checkstyle/*.xml') - } spotBugs { pattern('**/build/reports/spotbugs/*.xml') } diff --git a/.test-infra/jenkins/job_PreCommit_Java_AWS2_IO_Direct.groovy b/.test-infra/jenkins/job_PreCommit_Java_AWS2_IO_Direct.groovy index 2b734c227ec7..4d358e27e4ba 100644 --- a/.test-infra/jenkins/job_PreCommit_Java_AWS2_IO_Direct.groovy +++ b/.test-infra/jenkins/job_PreCommit_Java_AWS2_IO_Direct.groovy @@ -23,7 +23,8 @@ PrecommitJobBuilder builder = new PrecommitJobBuilder( nameBase: 'Java_Amazon-Web-Services2_IO_Direct', gradleTask: 'sdks:java:io:amazon-web-services2:integrationTest', gradleSwitches: [ - '-PdisableSpotlessCheck=true' + '-PdisableSpotlessCheck=true', + '-PdisableCheckStyle=true' ], // spotless checked in separate pre-commit triggerPathPatterns: [ '^sdks/java/io/amazon-web-services2/.*$', diff --git a/.test-infra/jenkins/job_PreCommit_Java_AWS_IO_Direct.groovy b/.test-infra/jenkins/job_PreCommit_Java_AWS_IO_Direct.groovy index d7e68c70b0da..618b0a1108a3 100644 --- a/.test-infra/jenkins/job_PreCommit_Java_AWS_IO_Direct.groovy +++ b/.test-infra/jenkins/job_PreCommit_Java_AWS_IO_Direct.groovy @@ -23,7 +23,8 @@ PrecommitJobBuilder builder = new PrecommitJobBuilder( nameBase: 'Java_Amazon-Web-Services_IO_Direct', gradleTask: ':sdks:java:io:amazon-web-services:integrationTest', gradleSwitches: [ - '-PdisableSpotlessCheck=true' + '-PdisableSpotlessCheck=true', + '-PdisableCheckStyle=true' ], // spotless checked in separate pre-commit triggerPathPatterns: [ '^sdks/java/io/amazon-web-services/.*$', diff --git a/.test-infra/jenkins/job_PreCommit_Java_Debezium_IO_Direct.groovy b/.test-infra/jenkins/job_PreCommit_Java_Debezium_IO_Direct.groovy index 8b42448faf99..ec157c8a1a70 100644 --- a/.test-infra/jenkins/job_PreCommit_Java_Debezium_IO_Direct.groovy +++ b/.test-infra/jenkins/job_PreCommit_Java_Debezium_IO_Direct.groovy @@ -23,7 +23,8 @@ PrecommitJobBuilder builder = new PrecommitJobBuilder( nameBase: 'Java_Debezium_IO_Direct', gradleTask: ':sdks:java:io:debezium:integrationTest', gradleSwitches: [ - '-PdisableSpotlessCheck=true' + '-PdisableSpotlessCheck=true', + '-PdisableCheckStyle=true' ], // spotless checked in separate pre-commit triggerPathPatterns: [ '^sdks/java/io/debezium/.*$', diff --git a/.test-infra/jenkins/job_PreCommit_Java_Examples_Dataflow.groovy b/.test-infra/jenkins/job_PreCommit_Java_Examples_Dataflow.groovy index bf692c4a7828..109456b3bc4f 100644 --- a/.test-infra/jenkins/job_PreCommit_Java_Examples_Dataflow.groovy +++ b/.test-infra/jenkins/job_PreCommit_Java_Examples_Dataflow.groovy @@ -23,7 +23,8 @@ PrecommitJobBuilder builder = new PrecommitJobBuilder( nameBase: 'Java_Examples_Dataflow', gradleTask: ':javaExamplesDataflowPreCommit', gradleSwitches: [ - '-PdisableSpotlessCheck=true' + '-PdisableSpotlessCheck=true', + '-PdisableCheckStyle=true' ], // spotless checked in separate pre-commit triggerPathPatterns: [ '^model/.*$', diff --git a/.test-infra/jenkins/job_PreCommit_Java_Examples_Dataflow_Java11.groovy b/.test-infra/jenkins/job_PreCommit_Java_Examples_Dataflow_Java11.groovy index 69473441a42e..adeb7f14c0eb 100644 --- a/.test-infra/jenkins/job_PreCommit_Java_Examples_Dataflow_Java11.groovy +++ b/.test-infra/jenkins/job_PreCommit_Java_Examples_Dataflow_Java11.groovy @@ -25,6 +25,7 @@ PrecommitJobBuilder builder = new PrecommitJobBuilder( gradleTask: ':clean', gradleSwitches: [ '-PdisableSpotlessCheck=true', + '-PdisableCheckStyle=true', '-PskipCheckerFramework' // Gradle itself is running under JDK8 so plugin configures wrong for JDK11 ], // spotless checked in separate pre-commit triggerPathPatterns: [ @@ -47,6 +48,7 @@ builder.build { rootBuildScriptDir(properties.checkoutDir) tasks 'javaExamplesDataflowPreCommit' switches '-PdisableSpotlessCheck=true' + switches '-PdisableCheckStyle=true' switches '-PskipCheckerFramework' // Gradle itself is running under JDK8 so plugin configures wrong for JDK11 switches '-PcompileAndRunTestsWithJava11' switches "-Pjava11Home=${properties.JAVA_11_HOME}" diff --git a/.test-infra/jenkins/job_PreCommit_Java_GCP_IO_Direct.groovy b/.test-infra/jenkins/job_PreCommit_Java_GCP_IO_Direct.groovy index c0a95954d772..b08dc08922db 100644 --- a/.test-infra/jenkins/job_PreCommit_Java_GCP_IO_Direct.groovy +++ b/.test-infra/jenkins/job_PreCommit_Java_GCP_IO_Direct.groovy @@ -23,7 +23,8 @@ PrecommitJobBuilder builder = new PrecommitJobBuilder( nameBase: 'Java_GCP_IO_Direct', gradleTask: ':sdks:java:io:google-cloud-platform:postCommit', gradleSwitches: [ - '-PdisableSpotlessCheck=true' + '-PdisableSpotlessCheck=true', + '-PdisableCheckStyle=true' ], // spotless checked in separate pre-commit timeoutMins: 120, triggerPathPatterns: [ diff --git a/.test-infra/jenkins/job_PreCommit_Java_JDBC_IO_Direct.groovy b/.test-infra/jenkins/job_PreCommit_Java_JDBC_IO_Direct.groovy index 123a30fac560..ca7d69d5efc7 100644 --- a/.test-infra/jenkins/job_PreCommit_Java_JDBC_IO_Direct.groovy +++ b/.test-infra/jenkins/job_PreCommit_Java_JDBC_IO_Direct.groovy @@ -23,7 +23,8 @@ PrecommitJobBuilder builder = new PrecommitJobBuilder( nameBase: 'Java_JDBC_IO_Direct', gradleTask: ':sdks:java:io:jdbc:integrationTest', gradleSwitches: [ - '-PdisableSpotlessCheck=true' + '-PdisableSpotlessCheck=true', + '-PdisableCheckStyle=true' ], // spotless checked in separate pre-commit triggerPathPatterns: [ '^sdks/java/io/jdbc/.*$', diff --git a/.test-infra/jenkins/job_PreCommit_Java_Kafka_IO_Direct.groovy b/.test-infra/jenkins/job_PreCommit_Java_Kafka_IO_Direct.groovy index 87c3044d4d2b..6869b53ca5cc 100644 --- a/.test-infra/jenkins/job_PreCommit_Java_Kafka_IO_Direct.groovy +++ b/.test-infra/jenkins/job_PreCommit_Java_Kafka_IO_Direct.groovy @@ -23,7 +23,8 @@ PrecommitJobBuilder builder = new PrecommitJobBuilder( nameBase: 'Java_Kafka_IO_Direct', gradleTask: ':sdks:java:io:kafka:kafkaVersionsCompatibilityTest', gradleSwitches: [ - '-PdisableSpotlessCheck=true' + '-PdisableSpotlessCheck=true', + '-PdisableCheckStyle=true' ], // spotless checked in separate pre-commit triggerPathPatterns: [ '^sdks/java/io/kafka/.*$', diff --git a/.test-infra/jenkins/job_PreCommit_Java_Kinesis_IO_Direct.groovy b/.test-infra/jenkins/job_PreCommit_Java_Kinesis_IO_Direct.groovy index cda54ba0867d..45861982cfd9 100644 --- a/.test-infra/jenkins/job_PreCommit_Java_Kinesis_IO_Direct.groovy +++ b/.test-infra/jenkins/job_PreCommit_Java_Kinesis_IO_Direct.groovy @@ -23,7 +23,8 @@ PrecommitJobBuilder builder = new PrecommitJobBuilder( nameBase: 'Java_Kinesis_IO_Direct', gradleTask: ':sdks:java:io:kinesis:integrationTest', gradleSwitches: [ - '-PdisableSpotlessCheck=true' + '-PdisableSpotlessCheck=true', + '-PdisableCheckStyle=true' ], // spotless checked in separate pre-commit triggerPathPatterns: [ '^sdks/java/io/kinesis/.*$', diff --git a/.test-infra/jenkins/job_PreCommit_Java_Neo4j_IO_Direct.groovy b/.test-infra/jenkins/job_PreCommit_Java_Neo4j_IO_Direct.groovy index 9042184bd655..b902c78899af 100644 --- a/.test-infra/jenkins/job_PreCommit_Java_Neo4j_IO_Direct.groovy +++ b/.test-infra/jenkins/job_PreCommit_Java_Neo4j_IO_Direct.groovy @@ -23,7 +23,8 @@ PrecommitJobBuilder builder = new PrecommitJobBuilder( nameBase: 'Java_Neo4j_IO_Direct', gradleTask: ':sdks:java:io:neo4j:integrationTest', gradleSwitches: [ - '-PdisableSpotlessCheck=true' + '-PdisableSpotlessCheck=true', + '-PdisableCheckStyle=true' ], // spotless checked in separate pre-commit triggerPathPatterns: [ '^sdks/java/io/neo4j/.*$', diff --git a/.test-infra/jenkins/job_PreCommit_SQL.groovy b/.test-infra/jenkins/job_PreCommit_SQL.groovy index c565cb99b567..241921ceae1a 100644 --- a/.test-infra/jenkins/job_PreCommit_SQL.groovy +++ b/.test-infra/jenkins/job_PreCommit_SQL.groovy @@ -23,7 +23,8 @@ PrecommitJobBuilder builder = new PrecommitJobBuilder( nameBase: 'SQL', gradleTask: ':sqlPreCommit', gradleSwitches: [ - '-PdisableSpotlessCheck=true' + '-PdisableSpotlessCheck=true', + '-PdisableCheckStyle=true' ], // spotless checked in job_PreCommit_Spotless triggerPathPatterns: [ '^sdks/java/extensions/sql.*$', @@ -35,10 +36,6 @@ builder.build { recordIssues { tools { errorProne() - java() - checkStyle { - pattern('**/build/reports/checkstyle/*.xml') - } spotBugs { pattern('**/build/reports/spotbugs/*.xml') } diff --git a/.test-infra/jenkins/job_PreCommit_SQL_Java11.groovy b/.test-infra/jenkins/job_PreCommit_SQL_Java11.groovy index 4521cf1b127e..9742ab756cf7 100644 --- a/.test-infra/jenkins/job_PreCommit_SQL_Java11.groovy +++ b/.test-infra/jenkins/job_PreCommit_SQL_Java11.groovy @@ -25,6 +25,7 @@ PrecommitJobBuilder builder = new PrecommitJobBuilder( gradleTask: ':sqlPreCommit', gradleSwitches: [ '-PdisableSpotlessCheck=true', + '-PdisableCheckStyle=true', '-PcompileAndRunTestsWithJava11', '-PskipCheckerFramework', // Gradle itself is running under JDK8 so plugin configures wrong for JDK11 @@ -41,9 +42,6 @@ builder.build { tools { errorProne() java() - checkStyle { - pattern('**/build/reports/checkstyle/*.xml') - } spotBugs { pattern('**/build/reports/spotbugs/*.xml') } diff --git a/.test-infra/jenkins/job_PreCommit_SQL_Java17.groovy b/.test-infra/jenkins/job_PreCommit_SQL_Java17.groovy index c70baad9dee6..158fa683c1a8 100644 --- a/.test-infra/jenkins/job_PreCommit_SQL_Java17.groovy +++ b/.test-infra/jenkins/job_PreCommit_SQL_Java17.groovy @@ -25,6 +25,7 @@ PrecommitJobBuilder builder = new PrecommitJobBuilder( gradleTask: ':sqlPreCommit', gradleSwitches: [ '-PdisableSpotlessCheck=true', + '-PdisableCheckStyle=true', '-PcompileAndRunTestsWithJava17', '-PskipCheckerFramework', // Gradle itself is running under JDK8 so plugin configures wrong for JDK17 @@ -40,9 +41,6 @@ builder.build { recordIssues { tools { java() - checkStyle { - pattern('**/build/reports/checkstyle/*.xml') - } spotBugs { pattern('**/build/reports/spotbugs/*.xml') } diff --git a/.test-infra/jenkins/job_PreCommit_Spotless.groovy b/.test-infra/jenkins/job_PreCommit_Spotless.groovy index 9b6ff37701b6..a9da1ad5491a 100644 --- a/.test-infra/jenkins/job_PreCommit_Spotless.groovy +++ b/.test-infra/jenkins/job_PreCommit_Spotless.groovy @@ -21,7 +21,7 @@ import PrecommitJobBuilder PrecommitJobBuilder builder = new PrecommitJobBuilder( scope: this, nameBase: 'Spotless', - gradleTask: 'spotlessCheck', + gradleTask: 'spotlessCheck checkStyleMain checkStyleTest', triggerPathPatterns: [ '^buildSrc/.*$', '^sdks/java/.*$', @@ -31,4 +31,15 @@ PrecommitJobBuilder builder = new PrecommitJobBuilder( '^.test-infra/jenkins/.*$', ] ) -builder.build() +builder.build { + publishers { + recordIssues { + tools { + checkStyle { + pattern('**/build/reports/checkstyle/*.xml') + } + } + enabledForFailure(true) + } + } +} diff --git a/.test-infra/jenkins/job_Precommit_Java_Examples_Dataflow_Java17.groovy b/.test-infra/jenkins/job_Precommit_Java_Examples_Dataflow_Java17.groovy index e1fc8b461071..29a37cf4b521 100644 --- a/.test-infra/jenkins/job_Precommit_Java_Examples_Dataflow_Java17.groovy +++ b/.test-infra/jenkins/job_Precommit_Java_Examples_Dataflow_Java17.groovy @@ -25,6 +25,7 @@ PrecommitJobBuilder builder = new PrecommitJobBuilder( gradleTask: ':clean', gradleSwitches: [ '-PdisableSpotlessCheck=true', + '-PdisableCheckStyle=true', '-PskipCheckerFramework' // Gradle itself is running under JDK8 so plugin configures wrong for JDK17 ], // spotless checked in separate pre-commit triggerPathPatterns: [ @@ -47,6 +48,7 @@ builder.build { rootBuildScriptDir(properties.checkoutDir) tasks 'javaExamplesDataflowPreCommit' switches '-PdisableSpotlessCheck=true' + switches '-PdisableCheckStyle=true' switches '-PskipCheckerFramework' // Gradle itself is running under JDK8 so plugin configures wrong for JDK17 switches '-PcompileAndRunTestsWithJava17' switches "-Pjava17Home=${properties.JAVA_17_HOME}" diff --git a/.test-infra/jenkins/job_sonarqube_report.groovy b/.test-infra/jenkins/job_sonarqube_report.groovy index afec03117e15..df55632621c9 100644 --- a/.test-infra/jenkins/job_sonarqube_report.groovy +++ b/.test-infra/jenkins/job_sonarqube_report.groovy @@ -45,7 +45,7 @@ job('beam_sonarqube_report') { tasks("sonarqube") switches("--continue") switches("-PdisableSpotlessCheck=true") - + switches("-PdisableCheckStyle=true") // disable parallelization to avoid output collisions switches("--no-parallel") } diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy index ae1c581c5547..3b01a8e243ab 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -1079,6 +1079,12 @@ class BeamModulePlugin implements Plugin { maxErrors = 0 } project.checkstyle { toolVersion = "8.23" } + // CheckStyle can be removed from the 'check' task by passing -PdisableCheckStyle=true on the Gradle + // command-line. This is useful for pre-commit which runs checkStyle separately. + def disableCheckStyle = project.hasProperty('disableCheckStyle') && + project.disableCheckStyle == 'true' + project.checkstyleMain.enabled = !disableCheckStyle + project.checkstyleTest.enabled = !disableCheckStyle // Configures javadoc plugin and ensure check runs javadoc. project.tasks.withType(Javadoc) {