Release pants as both a Python 2.7 PEX and a Python 3.6 PEX (#7401)

### Problem While wheels are now released with Python 3 support thanks to #7197, this is only one of two ways people consume Pants. The other way is through a Pex released to Github. We decided to release both a Py27 and Py36 Pex. We will not be releasing a Py37 Pex, as it is would require Py37 wheel building shards which would require a Centos7 docker image. We decided this is not worth the work for how few people we anticipate would use Py37 with Pex combined. We will also not be releasing a more flexible Pex that works with either Py27 or Py36, as it is blocked by pex-tool/pex#654. We also want organizations pinning their Python version, so a flexible release is not particularly useful in this instance. This implements #6450 (comment). ### Solution Release both `pants.${VERSION}.py{27,36}.pex`. This requires changing how we run `release.sh -p` and adding new CI shards to deploy on both Py2 and Py3. ### Result The next release, we will release both `pants.1.15.0.rc1.py27.pex` and `pants.1.15.0.rc1.py36.pex`.
pantsbuild · Mar 20, 2019 · d392a19 · d392a19
1 parent 8d64796
commit d392a19
Show file tree

Hide file tree

Showing 3 changed files with 154 additions and 42 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -703,17 +703,19 @@ py36_jvm_tests: &py36_jvm_tests
 # Deploy
 # -------------------------------------------------------------------------
 
-deploy_stable_multiplatform_pex: &deploy_stable_multiplatform_pex
-  name: "Deploy stable multiplatform pants.pex"
+base_deploy: &base_deploy
   os: linux
   language: python
+  env:
+    - &base_deploy_env RUN_PANTS_FROM_PEX=1
+
+base_deploy_stable_muliplatform_pex: &base_deploy_stable_muliplatform_pex
+  <<: *base_deploy
   stage: *build_stable
   env:
-    - CACHE_NAME=linuxpexdeploystable
-    - RUN_PANTS_FROM_PEX=1
-    - PANTS_PEX_RELEASE=stable
+    - &base_deploy_stable_env PANTS_PEX_RELEASE=stable
   script:
-    - ./build-support/bin/release.sh -p
+    - ./build-support/bin/release.sh -p ${RELEASE_ARGS}
   deploy:
     # See https://docs.travis-ci.com/user/deployment/releases/
     provider: releases
@@ -728,17 +730,55 @@ deploy_stable_multiplatform_pex: &deploy_stable_multiplatform_pex
       tags: true
       repo: pantsbuild/pants
 
-deploy_unstable_multiplatform_pex: &deploy_unstable_multiplatform_pex
-  name: "Deploy unstable multiplatform pants.pex"
-  os: linux
-  language: python
+py27_deploy_stable_multiplatform_pex: &py27_deploy_stable_multiplatform_pex
+  <<: *base_deploy_stable_muliplatform_pex
+  name: "Deploy stable multiplatform pants.pex (Py2.7 PEX)"
+  python: 2.7
+  env:
+    - *base_deploy_env
+    - *base_deploy_stable_env
+    - RELEASE_ARGS=""
+    - CACHE_NAME=linuxpexdeploystable.py27
+
+py36_deploy_stable_multiplatform_pex: &py36_deploy_stable_multiplatform_pex
+  <<: *base_deploy_stable_muliplatform_pex
+  name: "Deploy stable multiplatform pants.pex (Py3.6 PEX)"
+  python: 3.6
+  env:
+    - *base_deploy_env
+    - *base_deploy_stable_env
+    - RELEASE_ARGS="-3"
+    - CACHE_NAME=linuxpexdeploystable.py36
+
+base_deploy_unstable_multiplatform_pex: &base_deploy_unstable_multiplatform_pex
+  <<: *base_deploy_unstable_multiplatform_pex
   stage: *build_unstable
   env:
-    - CACHE_NAME=linuxpexdeployunstable
-    - RUN_PANTS_FROM_PEX=1
-    - PREPARE_DEPLOY=1
+    - &base_deploy_unstable_env PREPARE_DEPLOY=1
   script:
-    - ./build-support/bin/release.sh -p && mkdir -p dist/deploy/pex/ && mv dist/pants*.pex dist/deploy/pex/
+    - ./build-support/bin/release.sh -p ${RELEASE_ARGS}
+    - mkdir -p dist/deploy/pex/
+    - mv dist/pants*.pex dist/deploy/pex/
+
+py27_deploy_unstable_multiplatform_pex: &py27_deploy_unstable_multiplatform_pex
+  <<: *base_deploy_unstable_multiplatform_pex
+  name: "Deploy unstable multiplatform pants.pex (Py2.7 PEX)"
+  python: 2.7
+  env:
+    - *base_deploy_env
+    - *base_deploy_unstable_env
+    - RELEASE_ARGS=""
+    - CACHE_NAME=linuxpexdeployunstable.py27
+
+py36_deploy_unstable_multiplatform_pex: &py36_deploy_unstable_multiplatform_pex
+  <<: *base_deploy_unstable_multiplatform_pex
+  name: "Deploy unstable multiplatform pants.pex (Py3.6 PEX)"
+  python: 3.6
+  env:
+    - *base_deploy_env
+    - *base_deploy_unstable_env
+    - RELEASE_ARGS="-3"
+    - CACHE_NAME=linuxpexdeployunstable.py36
 
 # -------------------------------------------------------------------------
 # Test matrix
@@ -1142,8 +1182,11 @@ matrix:
     - <<: *py27_jvm_tests
     - <<: *py36_jvm_tests
 
-    - <<: *deploy_stable_multiplatform_pex
-    - <<: *deploy_unstable_multiplatform_pex
+    - <<: *py27_deploy_stable_multiplatform_pex
+    - <<: *py36_deploy_stable_multiplatform_pex
+
+    - <<: *py27_deploy_unstable_multiplatform_pex
+    - <<: *py36_deploy_unstable_multiplatform_pex
 
 deploy:
   # Deploy whatever a previous stage has left in dist/deploy.

diff --git a/build-support/bin/release.sh b/build-support/bin/release.sh
@@ -208,7 +208,18 @@ function build_3rdparty_packages() {
   start_travis_section "3rdparty" "Building 3rdparty whls from ${REQUIREMENTS_3RDPARTY_FILES[@]}"
   activate_tmp_venv
 
-  pip wheel --wheel-dir="${DEPLOY_3RDPARTY_WHEEL_DIR}/${version}" ${req_args}
+  wheel_args=("--wheel-dir=${DEPLOY_3RDPARTY_WHEEL_DIR}/${version}")
+
+  # TODO(pex#539): This code hacks around Pex's issue in resolving abi3 values with an abi
+  # from an earlier Python 3 version. Specifically, the cryptography wheel is normally marked as cp34-abi3,
+  # meaning it supports any Python >= 3.4. But when running `release.sh -3p`, Pex will fail to
+  # resolve the cryptography wheel we have. So, we work around this by building our own cryptography
+  # wheel with `cp36-cp36m`.
+  if [[ "${python_three:-false}" == "true" ]]; then
+    wheel_args=("--no-binary=cryptography" "${wheel_args[@]}")
+  fi
+
+  pip wheel "${wheel_args[@]}" ${req_args}
 
   deactivate
   end_travis_section
@@ -575,36 +586,51 @@ function build_pex() {
   local linux_platform_noabi="linux_x86_64"
   local osx_platform_noabi="macosx_10.11_x86_64"
 
+  dest_suffix="py27.pex"
+  if [[ "${python_three:-false}" == "true" ]]; then
+    dest_suffix="py36.pex"
+  fi
+
   case "${mode}" in
     build)
       case "$(uname)" in
         # NB: When building locally, we use a platform that does not refer to the ABI version, to
         # avoid needing to introspect the python ABI for this machine.
         Darwin)
-          local platforms=("${osx_platform_noabi}")
+          local platform=("${osx_platform_noabi}")
           ;;
         Linux)
-          local platforms=("${linux_platform_noabi}")
+          local platform=("${linux_platform_noabi}")
           ;;
         *)
           echo >&2 "Unknown uname"
           exit 1
           ;;
       esac
-      local dest="${ROOT}/dist/pants.${PANTS_UNSTABLE_VERSION}.${platform}.pex"
-      local stable_dest="${DEPLOY_DIR}/pex/pants.${PANTS_STABLE_VERSION}.${platform}.pex"
+      local platforms=("${platform}")
+      local dest="${ROOT}/dist/pants.${PANTS_UNSTABLE_VERSION}.${platform}.${dest_suffix}"
+      local stable_dest="${DEPLOY_DIR}/pex/pants.${PANTS_STABLE_VERSION}.${platform}.${dest_suffix}"
       ;;
     fetch)
-      # NB: We do not include Python 3 wheels, as we cannot release a Python 3 compatible PEX
-      # until https://github.com/pantsbuild/pex/issues/654 is fixed.
       local platforms=()
+      # Note once Pex can release flexible binaries (#654), we could release Pants as one big flexible Pex
+      # by consolidating the below ABIs into one entry. At the moment, we do not want to do this, as we expect
+      # organizations to pin their Python version. We also do not want the Pex to default to Py27 when the user
+      # would rather use Py3, as it will default to using the minimum acceptable interpreter discoverable.
+      # This decision may be worth revisiting once we drop Py2 support so that we instead release one universal
+      # Pex that works with any Python 3.6+ version.
+      abis=("cp-27-mu" "cp-27-m")
+      if [[ "${python_three:-false}" == "true" ]]; then
+        # To add Py37 support to the Pex, we will need to ensure we have Py37 wheels built and then add "cp-37-m" here.
+        abis=("cp-36-m")
+      fi
       for platform in "${linux_platform_noabi}" "${osx_platform_noabi}"; do
-        for abi in "cp-27-mu" "cp-27-m"; do
+        for abi in "${abis[@]}"; do
           platforms=("${platforms[@]}" "${platform}-${abi}")
         done
       done
-      local dest="${ROOT}/dist/pants.${PANTS_UNSTABLE_VERSION}.pex"
-      local stable_dest="${DEPLOY_DIR}/pex/pants.${PANTS_STABLE_VERSION}.pex"
+      local dest="${ROOT}/dist/pants.${PANTS_UNSTABLE_VERSION}.${dest_suffix}"
+      local stable_dest="${DEPLOY_DIR}/pex/pants.${PANTS_STABLE_VERSION}.${dest_suffix}"
       ;;
     *)
       echo >&2 "Bad build_pex mode ${mode}"

diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache
@@ -656,17 +656,19 @@ py36_jvm_tests: &py36_jvm_tests
 # Deploy
 # -------------------------------------------------------------------------
 
-deploy_stable_multiplatform_pex: &deploy_stable_multiplatform_pex
-  name: "Deploy stable multiplatform pants.pex"
+base_deploy: &base_deploy
   os: linux
   language: python
+  env:
+    - &base_deploy_env RUN_PANTS_FROM_PEX=1
+
+base_deploy_stable_muliplatform_pex: &base_deploy_stable_muliplatform_pex
+  <<: *base_deploy
   stage: *build_stable
   env:
-    - CACHE_NAME=linuxpexdeploystable
-    - RUN_PANTS_FROM_PEX=1
-    - PANTS_PEX_RELEASE=stable
+    - &base_deploy_stable_env PANTS_PEX_RELEASE=stable
   script:
-    - ./build-support/bin/release.sh -p
+    - ./build-support/bin/release.sh -p ${RELEASE_ARGS}
   deploy:
     # See https://docs.travis-ci.com/user/deployment/releases/
     provider: releases
@@ -681,17 +683,55 @@ deploy_stable_multiplatform_pex: &deploy_stable_multiplatform_pex
       tags: true
       repo: pantsbuild/pants
 
-deploy_unstable_multiplatform_pex: &deploy_unstable_multiplatform_pex
-  name: "Deploy unstable multiplatform pants.pex"
-  os: linux
-  language: python
+py27_deploy_stable_multiplatform_pex: &py27_deploy_stable_multiplatform_pex
+  <<: *base_deploy_stable_muliplatform_pex
+  name: "Deploy stable multiplatform pants.pex (Py2.7 PEX)"
+  python: 2.7
+  env:
+    - *base_deploy_env
+    - *base_deploy_stable_env
+    - RELEASE_ARGS=""
+    - CACHE_NAME=linuxpexdeploystable.py27
+
+py36_deploy_stable_multiplatform_pex: &py36_deploy_stable_multiplatform_pex
+  <<: *base_deploy_stable_muliplatform_pex
+  name: "Deploy stable multiplatform pants.pex (Py3.6 PEX)"
+  python: 3.6
+  env:
+    - *base_deploy_env
+    - *base_deploy_stable_env
+    - RELEASE_ARGS="-3"
+    - CACHE_NAME=linuxpexdeploystable.py36
+
+base_deploy_unstable_multiplatform_pex: &base_deploy_unstable_multiplatform_pex
+  <<: *base_deploy_unstable_multiplatform_pex
   stage: *build_unstable
   env:
-    - CACHE_NAME=linuxpexdeployunstable
-    - RUN_PANTS_FROM_PEX=1
-    - PREPARE_DEPLOY=1
+    - &base_deploy_unstable_env PREPARE_DEPLOY=1
   script:
-    - ./build-support/bin/release.sh -p && mkdir -p dist/deploy/pex/ && mv dist/pants*.pex dist/deploy/pex/
+    - ./build-support/bin/release.sh -p ${RELEASE_ARGS}
+    - mkdir -p dist/deploy/pex/
+    - mv dist/pants*.pex dist/deploy/pex/
+
+py27_deploy_unstable_multiplatform_pex: &py27_deploy_unstable_multiplatform_pex
+  <<: *base_deploy_unstable_multiplatform_pex
+  name: "Deploy unstable multiplatform pants.pex (Py2.7 PEX)"
+  python: 2.7
+  env:
+    - *base_deploy_env
+    - *base_deploy_unstable_env
+    - RELEASE_ARGS=""
+    - CACHE_NAME=linuxpexdeployunstable.py27
+
+py36_deploy_unstable_multiplatform_pex: &py36_deploy_unstable_multiplatform_pex
+  <<: *base_deploy_unstable_multiplatform_pex
+  name: "Deploy unstable multiplatform pants.pex (Py3.6 PEX)"
+  python: 3.6
+  env:
+    - *base_deploy_env
+    - *base_deploy_unstable_env
+    - RELEASE_ARGS="-3"
+    - CACHE_NAME=linuxpexdeployunstable.py36
 
 # -------------------------------------------------------------------------
 # Test matrix
@@ -805,8 +845,11 @@ matrix:
     - <<: *py27_jvm_tests
     - <<: *py36_jvm_tests
 
-    - <<: *deploy_stable_multiplatform_pex
-    - <<: *deploy_unstable_multiplatform_pex
+    - <<: *py27_deploy_stable_multiplatform_pex
+    - <<: *py36_deploy_stable_multiplatform_pex
+
+    - <<: *py27_deploy_unstable_multiplatform_pex
+    - <<: *py36_deploy_unstable_multiplatform_pex
 
 deploy:
   # Deploy whatever a previous stage has left in dist/deploy.