[BEAM-13183] [BEAM-8152] Use venv instead of virtualenv to create Pyt…

…hon environments in Gradle scripts. (#15819)
apache · Nov 18, 2021 · 3f504c7 · 3f504c7
1 parent 2335ec6
commit 3f504c7
Show file tree

Hide file tree

Showing 23 changed files with 97 additions and 76 deletions.
diff --git a/.test-infra/jenkins/CommonJobProperties.groovy b/.test-infra/jenkins/CommonJobProperties.groovy
@@ -22,12 +22,14 @@
 //  http://groovy-lang.org/style-guide.html
 
 import Committers as committers
+import PythonTestProperties as pythonTestProperties
 
 class CommonJobProperties {
 
   static String checkoutDir = 'src'
   final static String JAVA_8_HOME = '/usr/lib/jvm/java-8-openjdk-amd64'
   final static String JAVA_11_HOME = '/usr/lib/jvm/java-11-openjdk-amd64'
+  final static String PYTHON = pythonTestProperties.DEFAULT_INTERPRETER
 
   // Sets common top-level job properties for main repository jobs.
   static void setTopLevelMainJobProperties(def context,

diff --git a/.test-infra/jenkins/PythonTestProperties.groovy b/.test-infra/jenkins/PythonTestProperties.groovy
@@ -39,4 +39,7 @@ class PythonTestProperties {
   final static List<String> VALIDATES_CONTAINER_DATAFLOW_PYTHON_VERSIONS = ALL_SUPPORTED_VERSIONS
   final static String LOAD_TEST_PYTHON_VERSION = '3.7'
   final static String CHICAGO_TAXI_EXAMPLE_FLINK_PYTHON_VERSION = '3.7'
+  // Use for various shell scripts triggered by Jenkins.
+  // Gradle scripts should use project.ext.pythonVersion defined by PythonNature/BeamModulePlugin.
+  final static String DEFAULT_INTERPRETER = 'python3.8'
 }
diff --git a/.test-infra/jenkins/dependency_check/generate_report.sh b/.test-infra/jenkins/dependency_check/generate_report.sh
@@ -20,6 +20,9 @@
 set -e
 set -v
 
+# Get currently used Python version from args or assume a default.
+PYTHON=${1:-python3}
+
 REPORT_DESCRIPTION="
 <h4> A dependency update is high priority if it satisfies one of following criteria: </h4>
 <ul>
@@ -39,7 +42,8 @@ REPORT_DESCRIPTION="
 
 
 # Virtualenv for the rest of the script to run setup
-virtualenv dependency/check
+$PYTHON -m venv dependency/check
+
 . dependency/check/bin/activate
 pip install --upgrade google-cloud-bigquery
 pip install --upgrade google-cloud-bigtable
@@ -49,14 +53,14 @@ rm -f build/dependencyUpdates/beam-dependency-check-report.txt
 # Insall packages and run the unit tests of the report generator and the jira manager
 pip install mock jira pyyaml
 cd $WORKSPACE/src/.test-infra/jenkins
-python -m dependency_check.dependency_check_report_generator_test
-python -m jira_utils.jira_manager_test
-python -m dependency_check.version_comparer_test
+$PYTHON -m dependency_check.dependency_check_report_generator_test
+$PYTHON -m jira_utils.jira_manager_test
+$PYTHON -m dependency_check.version_comparer_test
 
 echo "<html><body>" > $WORKSPACE/src/build/dependencyUpdates/beam-dependency-check-report.html
 
-python -m dependency_check.dependency_check_report_generator Python
+$PYTHON -m dependency_check.dependency_check_report_generator Python
 
-python -m dependency_check.dependency_check_report_generator Java
+$PYTHON -m dependency_check.dependency_check_report_generator Java
 
 echo "$REPORT_DESCRIPTION </body></html>" >> $WORKSPACE/src/build/dependencyUpdates/beam-dependency-check-report.html
diff --git a/.test-infra/jenkins/job_Dependency_Check.groovy b/.test-infra/jenkins/job_Dependency_Check.groovy
@@ -47,7 +47,8 @@ job('beam_Dependency_Check') {
     }
 
     shell('cd ' + commonJobProperties.checkoutDir +
-        ' && bash .test-infra/jenkins/dependency_check/generate_report.sh')
+        ' && bash .test-infra/jenkins/dependency_check/generate_report.sh ' +
+        commonJobProperties.PYTHON)
   }
 
   wrappers{

diff --git a/.test-infra/jenkins/job_Inventory.groovy b/.test-infra/jenkins/job_Inventory.groovy
@@ -70,7 +70,7 @@ nums.each {
       shell('kubectl version || echo "kubectl not found"')
       ALL_SUPPORTED_VERSIONS.each { version ->
         def versionSuffix = version.replace('.', '')
-        shell("virtualenv -p python${version} test${versionSuffix} && . ./test${versionSuffix}/bin/activate && python --version && deactivate || echo \"python ${version} not found\"")
+        shell("python${version} -m venv test${versionSuffix} && . ./test${versionSuffix}/bin/activate && python --version && deactivate || echo \"python ${version} not found\"")
       }
       shell('echo "Maven home $MAVEN_HOME"')
       shell('env')

diff --git a/.test-infra/jupyter/README.md b/.test-infra/jupyter/README.md
@@ -22,10 +22,10 @@ test metrics.
 
 # Jupyter Setup
 
-Instructions for installing on Linux using pip+virtualenv:
+Instructions for installing on Linux using pip+venv:
 
 ```shell
-virtualenv --python python3 ~/virtualenvs/jupyter
+python3 -m venv ~/virtualenvs/jupyter
 source ~/virtualenvs/jupyter/bin/activate
 pip install jupyter
 # Optional packages, for example:

diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
@@ -2325,14 +2325,17 @@ class BeamModulePlugin implements Plugin<Project> {
       project.task('setupVirtualenv')  {
         doLast {
           def virtualenvCmd = [
-            'virtualenv',
+            "python${project.ext.pythonVersion}",
+            "-m",
+            "venv",
             "${project.ext.envdir}",
-            "--python=python${project.ext.pythonVersion}",
           ]
           project.exec { commandLine virtualenvCmd }
           project.exec {
             executable 'sh'
-            args '-c', ". ${project.ext.envdir}/bin/activate && pip install --retries 10 --upgrade tox==3.20.1 -r ${project.rootDir}/sdks/python/build-requirements.txt"
+            args '-c', ". ${project.ext.envdir}/bin/activate && " +
+                "pip install --retries 10 --upgrade pip && " +
+                "pip install --retries 10 --upgrade tox==3.20.1 -r ${project.rootDir}/sdks/python/build-requirements.txt"
           }
         }
         // Gradle will delete outputs whenever it thinks they are stale. Putting a

diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile
@@ -66,7 +66,7 @@ RUN alias python=python3.6
 ###
 # Install grpcio-tools mypy-protobuf for `python3 sdks/python/setup.py sdist` to work
 ###
-RUN pip3 install grpcio-tools mypy-protobuf virtualenv
+RUN pip3 install grpcio-tools mypy-protobuf
 
 ###
 # Install useful tools

diff --git a/dev-support/docker/pkglist b/dev-support/docker/pkglist
@@ -33,7 +33,19 @@ openjdk-8-jdk
 python3-setuptools
 python3-pip
 python3.6
+python3.6-dev
+python3.6-venv
 python3.7
+python3.7-dev
+python3.7-distutils
+python3.7-venv
 python3.8
+python3.8-dev
+python3.8-distutils
+python3.8-venv
+python3.9
+python3.9-dev
+python3.9-distutils
+python3.9-venv
 tox
 docker.io
diff --git a/local-env-setup.sh b/local-env-setup.sh
@@ -16,8 +16,8 @@
 # limitations under the License.
 
 darwin_install_pip3_packages() {
-    echo "Installing setuptools grpcio-tools virtualenv"
-    pip3 install setuptools grpcio-tools virtualenv
+    echo "Installing setuptools grpcio-tools"
+    pip3 install setuptools grpcio-tools
     echo "Installing mypy-protobuf"
     pip3 install --user mypy-protobuf
 }
@@ -48,8 +48,8 @@ if [ "$kernelname" = "Linux" ]; then
     type -P pip3 > /dev/null 2>&1
     pip3Exists=$?
     if [ $python3Exists -eq 0  -a $pip3Exists -eq 0 ]; then
-        echo "Installing grpcio-tools mypy-protobuf virtualenv"
-        pip3 install grpcio-tools mypy-protobuf virtualenv
+        echo "Installing grpcio-tools mypy-protobuf"
+        pip3 install grpcio-tools mypy-protobuf
     else
         echo "Python3 and pip3 are required but failed to install. Install them manually and rerun the script."
         exit
@@ -85,6 +85,22 @@ elif [ "$kernelname" = "Darwin" ]; then
         echo "Installing openjdk@8"
         brew install openjdk@8
     fi
+    for ver in 3.7 3.8 3.9; do
+      if brew ls --versions python@$ver > /dev/null; then
+          echo "python@$ver already installed. Skipping"
+          brew info python@$ver
+      else
+          echo "Installing python@$ver"
+          brew install python@$ver
+      fi
+      if [ ! $(type -P python$ver) > /dev/null 2>&1 ]; then
+          # For some python packages, brew does not add symlinks...
+          # TODO: Consider using pyenv to manage multiple installations of Python.
+          ln -s /usr/local/opt/python@$ver/bin/python3 /usr/local/bin/python$ver
+      fi
+    done
+
+    ls -l /usr/local/bin/python*
 
     type -P python3 > /dev/null 2>&1
     python3Exists=$?

diff --git a/release/src/main/python-release/python_release_automation_utils.sh b/release/src/main/python-release/python_release_automation_utils.sh
@@ -160,7 +160,7 @@ function install_sdk() {
   print_separator "Creating new virtualenv with $2 interpreter and installing the SDK from $sdk_file."
   gsutil version -l
   rm -rf ./temp_virtualenv_${2}
-  virtualenv temp_virtualenv_${2} -p $2
+  $2 -m venv temp_virtualenv_${2}
   . temp_virtualenv_${2}/bin/activate
   gcloud_version=$(gcloud --version | head -1 | awk '{print $4}')
   if [[ "$gcloud_version" < "189" ]]; then

diff --git a/release/src/main/scripts/deploy_pypi.sh b/release/src/main/scripts/deploy_pypi.sh
@@ -41,7 +41,7 @@ fi
 mkdir ${LOCAL_CLONE_DIR}
 cd ${LOCAL_CLONE_DIR}
 
-virtualenv deploy_pypi_env
+python3 -m venv deploy_pypi_env
 source ./deploy_pypi_env/bin/activate
 pip install twine
 

diff --git a/release/src/main/scripts/deploy_release_candidate_pypi.sh b/release/src/main/scripts/deploy_release_candidate_pypi.sh
@@ -127,11 +127,11 @@ RELEASE_COMMIT=$(git rev-list -n 1 $RC_TAG)
 
 echo "================Download python artifacts======================"
 PYTHON_ARTIFACTS_DIR="${LOCAL_CLONE_DIR_ROOT}/python"
-virtualenv deploy_pypi_env
+python3 -m venv deploy_pypi_env
 source ./deploy_pypi_env/bin/activate
 pip install -U pip
 pip install requests python-dateutil
-python "${SCRIPT_DIR}/download_github_actions_artifacts.py" \
+python3 "${SCRIPT_DIR}/download_github_actions_artifacts.py" \
   --github-user "${USER_GITHUB_ID}" \
   --repo-url "${GIT_REPO_BASE_URL}" \
   --rc-tag "${RC_TAG}" \

diff --git a/release/src/main/scripts/run_rc_validation.sh b/release/src/main/scripts/run_rc_validation.sh
@@ -378,7 +378,6 @@ if [[ ("$python_leaderboard_direct" = true \
 
   `which pip` install --upgrade pip
   `which pip` install --upgrade setuptools
-  `which pip` install --upgrade virtualenv
 
   echo "--------------------------Updating ~/.m2/settings.xml-------------------------"
     cd ~
@@ -444,7 +443,7 @@ if [[ ("$python_leaderboard_direct" = true \
   do
     rm -rf ./beam_env_${py_version}
     echo "--------------Setting up virtualenv with $py_version interpreter----------------"
-    virtualenv beam_env_${py_version} -p $py_version
+    $py_version -m venv beam_env_${py_version} 
     . beam_env_${py_version}/bin/activate
 
     echo "--------------------------Installing Python SDK-------------------------------"
@@ -621,7 +620,6 @@ if [[ ("$python_xlang_kafka_taxi_dataflow" = true
 
   `which pip` install --upgrade pip
   `which pip` install --upgrade setuptools
-  `which pip` install --upgrade virtualenv
 
   echo "-----------------------Setting up Shell Env Vars------------------------------"
   set_bashrc
@@ -648,7 +646,7 @@ if [[ ("$python_xlang_kafka_taxi_dataflow" = true
   do
     rm -rf ./beam_env_${py_version}
     echo "--------------Setting up virtualenv with $py_version interpreter----------------"
-    virtualenv beam_env_${py_version} -p $py_version
+    $py_version -m venv beam_env_${py_version}
     . beam_env_${py_version}/bin/activate
     ln -s ${LOCAL_BEAM_DIR}/sdks beam_env_${py_version}/lib/sdks
 

diff --git a/runners/portability/test_flink_uber_jar.sh b/runners/portability/test_flink_uber_jar.sh
@@ -80,7 +80,7 @@ docker -v
 docker images --format "{{.Repository}}:{{.Tag}}" | grep "$PYTHON_CONTAINER_IMAGE"
 
 # Set up Python environment
-virtualenv -p "python$PYTHON_VERSION" "$ENV_DIR"
+python$PYTHON_VERSION -m venv "$ENV_DIR"
 . $ENV_DIR/bin/activate
 pip install --retries 10 -e "$PYTHON_ROOT_DIR"
 

diff --git a/runners/portability/test_pipeline_jar.sh b/runners/portability/test_pipeline_jar.sh
@@ -72,7 +72,7 @@ echo "Checking for Docker image ${PYTHON_CONTAINER_IMAGE}"
 docker images --format "{{.Repository}}:{{.Tag}}" | grep $PYTHON_CONTAINER_IMAGE
 
 # Set up Python environment
-virtualenv -p python$PYTHON_VERSION $ENV_DIR
+python$PYTHON_VERSION -m venv $ENV_DIR
 . $ENV_DIR/bin/activate
 pip install --retries 10 -e $PYTHON_ROOT_DIR
 

diff --git a/sdks/java/container/build.gradle b/sdks/java/container/build.gradle
@@ -22,6 +22,7 @@ plugins {
 }
 
 applyGoNature()
+applyPythonNature()
 
 evaluationDependsOn(":sdks:java:harness")
 evaluationDependsOn(":sdks:java:io:kafka")
@@ -92,7 +93,7 @@ task pullLicenses(type: Exec) {
   dependsOn generateLicenseReport
   generateLicenseReport.outputs.cacheIf { false }
 
-  commandLine './license_scripts/license_script.sh'
+  commandLine './license_scripts/license_script.sh', "python${project.ext.pythonVersion}"
   outputs.dir('build/target/third_party_licenses')
   outputs.dir('build/target/java_third_party_licenses')
 }

diff --git a/sdks/java/container/license_scripts/license_script.sh b/sdks/java/container/license_scripts/license_script.sh
@@ -16,6 +16,8 @@
 
 set -e
 
+# Get currently used Python version from Gradle or assume a default.
+PYTHON=${1:-python3}
 SCRIPT_DIR="${PWD}/license_scripts"
 ENV_DIR="${PWD}/build/virtualenv"
 
@@ -38,8 +40,7 @@ if [ -d "$DOWNLOAD_DIR" ]; then rm -rf "$DOWNLOAD_DIR" ; fi
 mkdir -p "$DOWNLOAD_DIR"
 cp -r "${EXISTING_LICENSE_DIR}"/*.jar "${DOWNLOAD_DIR}"
 
-# activate virtualenv
-virtualenv --python=python3 ${ENV_DIR} && . ${ENV_DIR}/bin/activate
+$PYTHON -m venv ${ENV_DIR} && . ${ENV_DIR}/bin/activate
 
 # install packages
 ${ENV_DIR}/bin/pip install -r ${SCRIPT_DIR}/requirement.txt
@@ -61,4 +62,4 @@ cp -r "$DOWNLOAD_DIR"/*.jar "$DEST_DIR"/
 cp -r "$DOWNLOAD_DIR"/*.csv "$DEST_DIR"/
 gzip -r "$DEST_DIR"/*
 
-echo "Finished license_scripts.sh"
+echo "Finished license_scripts.sh"
diff --git a/sdks/python/build-requirements.txt b/sdks/python/build-requirements.txt
@@ -14,6 +14,12 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #
+
+# TODO(BEAM-8954): Consider PEP-517/PEP-518 instead of this file.
+
+setuptools
+wheel>=0.36.0
+
 grpcio-tools==1.37.0
 mypy-protobuf==1.18
 

diff --git a/website/build.gradle b/website/build.gradle
@@ -21,6 +21,9 @@ plugins {
   id 'org.ajoberstar.grgit'  // Publish website to asf-git branch.
 }
 
+apply plugin: org.apache.beam.gradle.BeamModulePlugin
+applyPythonNature()
+
 def dockerImageTag = 'beam-website'
 def dockerWorkDir = "/opt"
 def buildDir = "${project.rootDir}/build/website"
@@ -41,10 +44,10 @@ def shell = { cmd ->
 
 def envdir = "${buildDir}/gradleenv"
 
-task setupVirtualenv {
+task setupVirtualenvWithDependencies {
   doLast {
     exec {
-      commandLine 'virtualenv', "${envdir}"
+      commandLine "python${project.ext.pythonVersion}", '-m', 'venv', "${envdir}"
     }
     exec {
       executable 'sh'
@@ -360,5 +363,5 @@ task stageWebsite {
   }
 }
 
-stageWebsite.dependsOn setupVirtualenv
+stageWebsite.dependsOn setupVirtualenvWithDependencies
 stageWebsite.dependsOn buildGcsWebsite
diff --git a/website/www/site/content/en/contribute/_index.md b/website/www/site/content/en/contribute/_index.md
@@ -80,8 +80,9 @@ detail.
    changes locally.
  - For SDK Development:
       - [Go](https://golang.org) 1.12 or later installed for Go SDK development
- - Python 3.6, 3.7, and 3.8. Yes, you need all three versions installed.
-      - pip, setuptools, virtualenv, and tox installed for Python development
+      - Python 3.x interpreters. You will need Python interpreters for all Python versions supported by Beam.
+        Interpreters should be installed and available in shell via `python3.x` commands. For more information, see:
+        Python installation tips in [Developer Wiki](https://cwiki.apache.org/confluence/display/BEAM/Python+Tips#PythonTips-InstallingPythoninterpreters).
  - For large contributions, a signed [Individual Contributor License
    Agreement](https://www.apache.org/licenses/icla.pdf) (ICLA) to the Apache
    Software Foundation (ASF).