Merge pull request apache#17354 from [BEAM-14170] - Create a test tha…

…t runs sickbayed tests ( Python Sickbay tests) * Add sickbast task and groovy job * Fix typo * Delete comment unit test skip * Delete jar for dataflow, not needed * Fix task name * Added the new job to the read me, fix typo, create a task on build gradle kts * Fix Pylint line * Resolve Merge conflict
AnandInguva · Apr 26, 2022 · 0654109 · 0654109
1 parent 429afcd
commit 0654109
Show file tree

Hide file tree

Showing 6 changed files with 93 additions and 4 deletions.
diff --git a/.test-infra/jenkins/README.md b/.test-infra/jenkins/README.md
@@ -120,6 +120,9 @@ Beam Jenkins overview page: [link](https://ci-beam.apache.org/)
 | beam_PostCommit_Python_VR_Spark | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Python_VR_Spark/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Python_VR_Spark/) | `Run Python Spark ValidatesRunner` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Python_VR_Spark/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Python_VR_Spark) |
 | beam_PostCommit_Python37 | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Python37), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Python37_PR/) | `Run Python 3.7 PostCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Python37/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Python37) |
 | beam_PostCommit_Python38 | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Python38), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_Python38_PR/) | `Run Python 3.8 PostCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Python38/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Python38) |
+| beam_PostCommit_Sickbay_Python36 | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Sickbay_Python36), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_SickBay_Python36_PR/) | `Run Python 3.6 PostCommit Sickbay tests` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Sickbay_Python36/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Sickbay_Python36) |
+| beam_PostCommit_Sickbay_Python37 | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Sickbay_Python37), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_SickBay_Python37_PR/) | `Run Python 3.7 PostCommit Sickbay tests` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Sickbay_Python37/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Sickbay_Python37) |
+| beam_PostCommit_Sickbay_Python38 | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Sickbay_Python38), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_SickBay_Python38_PR/) | `Run Python 3.8 PostCommit Sickbay tests` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Sickbay_Python38/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Sickbay_Python38) |
 | beam_PostCommit_SQL | [cron](https://ci-beam.apache.org/job/beam_PostCommit_SQL/), [phrase](https://ci-beam.apache.org/job/beam_PostCommit_SQL_PR/) | `Run SQL PostCommit` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_SQL/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_SQL) |
 | beam_PostCommit_Website_Publish | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Website_Publish/) | N/A | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Website_Publish/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Website_Publish) |
 | beam_PostCommit_Website_Test | [cron](https://ci-beam.apache.org/job/beam_PostCommit_Website_Test/) | `Run Full Website Test` | [![Build Status](https://ci-beam.apache.org/job/beam_PostCommit_Website_Test/badge/icon)](https://ci-beam.apache.org/job/beam_PostCommit_Website_Test) |

diff --git a/.test-infra/jenkins/job_PostCommit_Python_Sickbay.groovy b/.test-infra/jenkins/job_PostCommit_Python_Sickbay.groovy
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import CommonJobProperties as commonJobProperties
+import PostcommitJobBuilder
+
+import static PythonTestProperties.ALL_SUPPORTED_VERSIONS
+
+// This job defines the Python postcommit sickbay tests.
+ALL_SUPPORTED_VERSIONS.each { pythonVersion ->
+  def versionSuffix = pythonVersion.replace('.', '')
+  PostcommitJobBuilder.postCommitJob("beam_PostCommit_Sickbay_Python${versionSuffix}",
+      "Run Python ${pythonVersion} PostCommit Sickbay",
+      "Python${versionSuffix}_PC_Sickbay(\"Run Python ${pythonVersion} PostCommit Sickbay\")", this) {
+        description('Runs Python postcommit sickbay tests using Python ${pythonVersion}.')
+
+        // Set common parameters.
+        commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 180)
+
+        publishers {
+          archiveJunit('**/pytest*.xml')
+        }
+
+        // Execute shell command to test Python SDK.
+        steps {
+          gradle {
+            rootBuildScriptDir(commonJobProperties.checkoutDir)
+            tasks(":python${versionSuffix}SickbayPostCommit")
+            commonJobProperties.setGradleSwitches(delegate)
+          }
+        }
+      }
+}
diff --git a/build.gradle.kts b/build.gradle.kts
@@ -321,6 +321,23 @@ tasks.register("python39PostCommit") {
   dependsOn(":sdks:python:test-suites:portable:py39:postCommitPy39")
 }
 
+
+task("python36SickbayPostCommit") {
+  dependsOn(":sdks:python:test-suites:dataflow:py36:postCommitSickbay")
+}
+
+task("python37SickbayPostCommit") {
+  dependsOn(":sdks:python:test-suites:dataflow:py37:postCommitSickbay")
+}
+
+task("python38SickbayPostCommit") {
+  dependsOn(":sdks:python:test-suites:dataflow:py38:postCommitSickbay")
+}
+
+task("python39SickbayPostCommit") {
+  dependsOn(":sdks:python:test-suites:dataflow:py39:postCommitSickbay")
+}
+
 tasks.register("portablePythonPreCommit") {
   dependsOn(":sdks:python:test-suites:portable:py37:preCommitPy37")
   dependsOn(":sdks:python:test-suites:portable:py39:preCommitPy39")

diff --git a/sdks/python/apache_beam/io/gcp/gcsio_integration_test.py b/sdks/python/apache_beam/io/gcp/gcsio_integration_test.py
@@ -121,8 +121,8 @@ def test_copy_kms(self):
       raise unittest.SkipTest('--kms_key_name not specified')
     self._test_copy("test_copy_kms", self.kms_key_name)
 
-  @pytest.mark.it_postcommit
-  @unittest.skip('BEAM-12352: enable once maxBytesRewrittenPerCall works again')
+  @pytest.mark.it_postcommit_sickbay
+  #@pytest.mark.it_postcommit BEAM-12352 maxBytesRewrittenPerCall '
   def test_copy_rewrite_token(self):
     # Tests a multi-part copy (rewrite) operation. This is triggered by a
     # combination of 3 conditions:
@@ -175,8 +175,8 @@ def test_copy_batch_kms(self):
       raise unittest.SkipTest('--kms_key_name not specified')
     self._test_copy_batch("test_copy_batch_kms", self.kms_key_name)
 
-  @pytest.mark.it_postcommit
-  @unittest.skip('BEAM-12352: enable once maxBytesRewrittenPerCall works again')
+  @pytest.mark.it_postcommit_sickbay
+  #@pytest.mark.it_postcommit BEAM-12352  maxBytesRewrittenPerCall
   def test_copy_batch_rewrite_token(self):
     # Tests a multi-part copy (rewrite) operation. This is triggered by a
     # combination of 3 conditions:

diff --git a/sdks/python/pytest.ini b/sdks/python/pytest.ini
@@ -30,6 +30,7 @@ markers =
     xlang_transforms: collect Cross Language transforms test runs
     xlang_sql_expansion_service: collect for Cross Language with SQL expansion service test runs
     it_postcommit: collect for post-commit integration test runs
+    it_postcommit_sickbay: collect for post-commit sickbay integration test run
     it_validatesrunner: collect for ValidatesRunner integration test runs
     spannerio_it : collect for Spanner IO IT test runs.
     no_sickbay_streaming: run without sickbay-streaming

diff --git a/sdks/python/test-suites/dataflow/common.gradle b/sdks/python/test-suites/dataflow/common.gradle
@@ -125,6 +125,26 @@ task postCommitIT {
   }
 }
 
+task postCommitSickbay {
+  dependsOn 'installGcpTest'
+  dependsOn ':sdks:python:sdist'
+
+  doLast {
+    def testOpts = basicPytestOpts + ["--numprocesses=8", "--dist=loadfile"]
+    def argMap = [
+        "test_opts": testOpts,
+        "sdk_location": files(configurations.distTarBall.files).singleFile,
+        "suite": "postCommitIT-df${pythonVersionSuffix}",
+        "collect": "it_postcommit_sickbay"
+    ]
+    def cmdArgs = mapToArgString(argMap)
+    exec {
+      executable 'sh'
+      args '-c', ". ${envdir}/bin/activate && ${runScriptsDir}/run_integration_test.sh $cmdArgs"
+    }
+  }
+}
+
 task spannerioIT {
   dependsOn 'installGcpTest'
   dependsOn ':sdks:python:sdist'