diff --git a/.test-infra/jenkins/README.md b/.test-infra/jenkins/README.md index ab59c0c34f68..131d2dde0f54 100644 --- a/.test-infra/jenkins/README.md +++ b/.test-infra/jenkins/README.md @@ -120,6 +120,9 @@ Beam Jenkins overview page: [link](https://ci-beam.apache.org/) | beam_PostCommit_Python_VR_Spark | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Python_VR_Spark/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Python_VR_Spark/) | `Run Python Spark ValidatesRunner` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Python_VR_Spark/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Python_VR_Spark) | | beam_PostCommit_Python37 | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Python37), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Python37_PR/) | `Run Python 3.7 PostCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Python37/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Python37) | | beam_PostCommit_Python38 | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Python38), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Python38_PR/) | `Run Python 3.8 PostCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Python38/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Python38) | +| beam_PostCommit_Sickbay_Python36 | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Sickbay_Python36), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_SickBay_Python36_PR/) | `Run Python 3.6 PostCommit Sickbay tests` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Sickbay_Python36/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Sickbay_Python36) | +| beam_PostCommit_Sickbay_Python37 | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Sickbay_Python37), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_SickBay_Python37_PR/) | `Run Python 3.7 PostCommit Sickbay tests` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Sickbay_Python37/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Sickbay_Python37) | +| beam_PostCommit_Sickbay_Python38 | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Sickbay_Python38), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_SickBay_Python38_PR/) | `Run Python 3.8 PostCommit Sickbay tests` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Sickbay_Python38/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Sickbay_Python38) | | beam_PostCommit_SQL | [cron](https://ci-beam.apache.org/job/beam_PostCommit_SQL/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_SQL_PR/) | `Run SQL PostCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_SQL/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_SQL) | | beam_PostCommit_Website_Publish | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Website_Publish/) | N/A | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Website_Publish/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Website_Publish) | | beam_PostCommit_Website_Test | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Website_Test/) | `Run Full Website Test` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Website_Test/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Website_Test) | diff --git a/.test-infra/jenkins/job_PostCommit_Python_Sickbay.groovy b/.test-infra/jenkins/job_PostCommit_Python_Sickbay.groovy new file mode 100644 index 000000000000..9789121ee77f --- /dev/null +++ b/.test-infra/jenkins/job_PostCommit_Python_Sickbay.groovy @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import CommonJobProperties as commonJobProperties +import PostcommitJobBuilder + +import static PythonTestProperties.ALL_SUPPORTED_VERSIONS + +// This job defines the Python postcommit sickbay tests. +ALL_SUPPORTED_VERSIONS.each { pythonVersion -> + def versionSuffix = pythonVersion.replace('.', '') + PostcommitJobBuilder.postCommitJob("beam_PostCommit_Sickbay_Python${versionSuffix}", + "Run Python ${pythonVersion} PostCommit Sickbay", + "Python${versionSuffix}_PC_Sickbay(\"Run Python ${pythonVersion} PostCommit Sickbay\")", this) { + description('Runs Python postcommit sickbay tests using Python ${pythonVersion}.') + + // Set common parameters. + commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 180) + + publishers { + archiveJunit('**/pytest*.xml') + } + + // Execute shell command to test Python SDK. + steps { + gradle { + rootBuildScriptDir(commonJobProperties.checkoutDir) + tasks(":python${versionSuffix}SickbayPostCommit") + commonJobProperties.setGradleSwitches(delegate) + } + } + } +} \ No newline at end of file diff --git a/build.gradle.kts b/build.gradle.kts index 7b31c9db20ce..e59cbf29c533 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -321,6 +321,23 @@ tasks.register("python39PostCommit") { dependsOn(":sdks:python:test-suites:portable:py39:postCommitPy39") } + +task("python36SickbayPostCommit") { + dependsOn(":sdks:python:test-suites:dataflow:py36:postCommitSickbay") +} + +task("python37SickbayPostCommit") { + dependsOn(":sdks:python:test-suites:dataflow:py37:postCommitSickbay") +} + +task("python38SickbayPostCommit") { + dependsOn(":sdks:python:test-suites:dataflow:py38:postCommitSickbay") +} + +task("python39SickbayPostCommit") { + dependsOn(":sdks:python:test-suites:dataflow:py39:postCommitSickbay") +} + tasks.register("portablePythonPreCommit") { dependsOn(":sdks:python:test-suites:portable:py37:preCommitPy37") dependsOn(":sdks:python:test-suites:portable:py39:preCommitPy39") diff --git a/sdks/python/apache_beam/io/gcp/gcsio_integration_test.py b/sdks/python/apache_beam/io/gcp/gcsio_integration_test.py index e13d993b0e80..bf67273bad0f 100644 --- a/sdks/python/apache_beam/io/gcp/gcsio_integration_test.py +++ b/sdks/python/apache_beam/io/gcp/gcsio_integration_test.py @@ -121,8 +121,8 @@ def test_copy_kms(self): raise unittest.SkipTest('--kms_key_name not specified') self._test_copy("test_copy_kms", self.kms_key_name) - @pytest.mark.it_postcommit - @unittest.skip('BEAM-12352: enable once maxBytesRewrittenPerCall works again') + @pytest.mark.it_postcommit_sickbay + #@pytest.mark.it_postcommit BEAM-12352 maxBytesRewrittenPerCall ' def test_copy_rewrite_token(self): # Tests a multi-part copy (rewrite) operation. This is triggered by a # combination of 3 conditions: @@ -175,8 +175,8 @@ def test_copy_batch_kms(self): raise unittest.SkipTest('--kms_key_name not specified') self._test_copy_batch("test_copy_batch_kms", self.kms_key_name) - @pytest.mark.it_postcommit - @unittest.skip('BEAM-12352: enable once maxBytesRewrittenPerCall works again') + @pytest.mark.it_postcommit_sickbay + #@pytest.mark.it_postcommit BEAM-12352 maxBytesRewrittenPerCall def test_copy_batch_rewrite_token(self): # Tests a multi-part copy (rewrite) operation. This is triggered by a # combination of 3 conditions: diff --git a/sdks/python/pytest.ini b/sdks/python/pytest.ini index f24c9a9352e1..c25425c5208c 100644 --- a/sdks/python/pytest.ini +++ b/sdks/python/pytest.ini @@ -30,6 +30,7 @@ markers = xlang_transforms: collect Cross Language transforms test runs xlang_sql_expansion_service: collect for Cross Language with SQL expansion service test runs it_postcommit: collect for post-commit integration test runs + it_postcommit_sickbay: collect for post-commit sickbay integration test run it_validatesrunner: collect for ValidatesRunner integration test runs spannerio_it : collect for Spanner IO IT test runs. no_sickbay_streaming: run without sickbay-streaming diff --git a/sdks/python/test-suites/dataflow/common.gradle b/sdks/python/test-suites/dataflow/common.gradle index 4a66bef46989..fdc428001c3b 100644 --- a/sdks/python/test-suites/dataflow/common.gradle +++ b/sdks/python/test-suites/dataflow/common.gradle @@ -125,6 +125,26 @@ task postCommitIT { } } +task postCommitSickbay { + dependsOn 'installGcpTest' + dependsOn ':sdks:python:sdist' + + doLast { + def testOpts = basicPytestOpts + ["--numprocesses=8", "--dist=loadfile"] + def argMap = [ + "test_opts": testOpts, + "sdk_location": files(configurations.distTarBall.files).singleFile, + "suite": "postCommitIT-df${pythonVersionSuffix}", + "collect": "it_postcommit_sickbay" + ] + def cmdArgs = mapToArgString(argMap) + exec { + executable 'sh' + args '-c', ". ${envdir}/bin/activate && ${runScriptsDir}/run_integration_test.sh $cmdArgs" + } + } +} + task spannerioIT { dependsOn 'installGcpTest' dependsOn ':sdks:python:sdist'