diff --git a/.travis.yml b/.travis.yml index 3d632e69f94..937c2bab328 100644 --- a/.travis.yml +++ b/.travis.yml @@ -703,17 +703,19 @@ py36_jvm_tests: &py36_jvm_tests # Deploy # ------------------------------------------------------------------------- -deploy_stable_multiplatform_pex: &deploy_stable_multiplatform_pex - name: "Deploy stable multiplatform pants.pex" +base_deploy: &base_deploy os: linux language: python + env: + - &base_deploy_env RUN_PANTS_FROM_PEX=1 + +base_deploy_stable_muliplatform_pex: &base_deploy_stable_muliplatform_pex + <<: *base_deploy stage: *build_stable env: - - CACHE_NAME=linuxpexdeploystable - - RUN_PANTS_FROM_PEX=1 - - PANTS_PEX_RELEASE=stable + - &base_deploy_stable_env PANTS_PEX_RELEASE=stable script: - - ./build-support/bin/release.sh -p + - ./build-support/bin/release.sh -p ${RELEASE_ARGS} deploy: # See https://docs.travis-ci.com/user/deployment/releases/ provider: releases @@ -728,17 +730,55 @@ deploy_stable_multiplatform_pex: &deploy_stable_multiplatform_pex tags: true repo: pantsbuild/pants -deploy_unstable_multiplatform_pex: &deploy_unstable_multiplatform_pex - name: "Deploy unstable multiplatform pants.pex" - os: linux - language: python +py27_deploy_stable_multiplatform_pex: &py27_deploy_stable_multiplatform_pex + <<: *base_deploy_stable_muliplatform_pex + name: "Deploy stable multiplatform pants.pex (Py2.7 PEX)" + python: 2.7 + env: + - *base_deploy_env + - *base_deploy_stable_env + - RELEASE_ARGS="" + - CACHE_NAME=linuxpexdeploystable.py27 + +py36_deploy_stable_multiplatform_pex: &py36_deploy_stable_multiplatform_pex + <<: *base_deploy_stable_muliplatform_pex + name: "Deploy stable multiplatform pants.pex (Py3.6 PEX)" + python: 3.6 + env: + - *base_deploy_env + - *base_deploy_stable_env + - RELEASE_ARGS="-3" + - CACHE_NAME=linuxpexdeploystable.py36 + +base_deploy_unstable_multiplatform_pex: &base_deploy_unstable_multiplatform_pex + <<: *base_deploy_unstable_multiplatform_pex stage: *build_unstable env: - - CACHE_NAME=linuxpexdeployunstable - - RUN_PANTS_FROM_PEX=1 - - PREPARE_DEPLOY=1 + - &base_deploy_unstable_env PREPARE_DEPLOY=1 script: - - ./build-support/bin/release.sh -p && mkdir -p dist/deploy/pex/ && mv dist/pants*.pex dist/deploy/pex/ + - ./build-support/bin/release.sh -p ${RELEASE_ARGS} + - mkdir -p dist/deploy/pex/ + - mv dist/pants*.pex dist/deploy/pex/ + +py27_deploy_unstable_multiplatform_pex: &py27_deploy_unstable_multiplatform_pex + <<: *base_deploy_unstable_multiplatform_pex + name: "Deploy unstable multiplatform pants.pex (Py2.7 PEX)" + python: 2.7 + env: + - *base_deploy_env + - *base_deploy_unstable_env + - RELEASE_ARGS="" + - CACHE_NAME=linuxpexdeployunstable.py27 + +py36_deploy_unstable_multiplatform_pex: &py36_deploy_unstable_multiplatform_pex + <<: *base_deploy_unstable_multiplatform_pex + name: "Deploy unstable multiplatform pants.pex (Py3.6 PEX)" + python: 3.6 + env: + - *base_deploy_env + - *base_deploy_unstable_env + - RELEASE_ARGS="-3" + - CACHE_NAME=linuxpexdeployunstable.py36 # ------------------------------------------------------------------------- # Test matrix @@ -1142,8 +1182,11 @@ matrix: - <<: *py27_jvm_tests - <<: *py36_jvm_tests - - <<: *deploy_stable_multiplatform_pex - - <<: *deploy_unstable_multiplatform_pex + - <<: *py27_deploy_stable_multiplatform_pex + - <<: *py36_deploy_stable_multiplatform_pex + + - <<: *py27_deploy_unstable_multiplatform_pex + - <<: *py36_deploy_unstable_multiplatform_pex deploy: # Deploy whatever a previous stage has left in dist/deploy. diff --git a/build-support/bin/release.sh b/build-support/bin/release.sh index 93e0ff02ffa..8c7a4745667 100755 --- a/build-support/bin/release.sh +++ b/build-support/bin/release.sh @@ -208,7 +208,18 @@ function build_3rdparty_packages() { start_travis_section "3rdparty" "Building 3rdparty whls from ${REQUIREMENTS_3RDPARTY_FILES[@]}" activate_tmp_venv - pip wheel --wheel-dir="${DEPLOY_3RDPARTY_WHEEL_DIR}/${version}" ${req_args} + wheel_args=("--wheel-dir=${DEPLOY_3RDPARTY_WHEEL_DIR}/${version}") + + # TODO(pex#539): This code hacks around Pex's issue in resolving abi3 values with an abi + # from an earlier Python 3 version. Specifically, the cryptography wheel is normally marked as cp34-abi3, + # meaning it supports any Python >= 3.4. But when running `release.sh -3p`, Pex will fail to + # resolve the cryptography wheel we have. So, we work around this by building our own cryptography + # wheel with `cp36-cp36m`. + if [[ "${python_three:-false}" == "true" ]]; then + wheel_args=("--no-binary=cryptography" "${wheel_args[@]}") + fi + + pip wheel "${wheel_args[@]}" ${req_args} deactivate end_travis_section @@ -575,36 +586,51 @@ function build_pex() { local linux_platform_noabi="linux_x86_64" local osx_platform_noabi="macosx_10.11_x86_64" + dest_suffix="py27.pex" + if [[ "${python_three:-false}" == "true" ]]; then + dest_suffix="py36.pex" + fi + case "${mode}" in build) case "$(uname)" in # NB: When building locally, we use a platform that does not refer to the ABI version, to # avoid needing to introspect the python ABI for this machine. Darwin) - local platforms=("${osx_platform_noabi}") + local platform=("${osx_platform_noabi}") ;; Linux) - local platforms=("${linux_platform_noabi}") + local platform=("${linux_platform_noabi}") ;; *) echo >&2 "Unknown uname" exit 1 ;; esac - local dest="${ROOT}/dist/pants.${PANTS_UNSTABLE_VERSION}.${platform}.pex" - local stable_dest="${DEPLOY_DIR}/pex/pants.${PANTS_STABLE_VERSION}.${platform}.pex" + local platforms=("${platform}") + local dest="${ROOT}/dist/pants.${PANTS_UNSTABLE_VERSION}.${platform}.${dest_suffix}" + local stable_dest="${DEPLOY_DIR}/pex/pants.${PANTS_STABLE_VERSION}.${platform}.${dest_suffix}" ;; fetch) - # NB: We do not include Python 3 wheels, as we cannot release a Python 3 compatible PEX - # until https://github.com/pantsbuild/pex/issues/654 is fixed. local platforms=() + # Note once Pex can release flexible binaries (#654), we could release Pants as one big flexible Pex + # by consolidating the below ABIs into one entry. At the moment, we do not want to do this, as we expect + # organizations to pin their Python version. We also do not want the Pex to default to Py27 when the user + # would rather use Py3, as it will default to using the minimum acceptable interpreter discoverable. + # This decision may be worth revisiting once we drop Py2 support so that we instead release one universal + # Pex that works with any Python 3.6+ version. + abis=("cp-27-mu" "cp-27-m") + if [[ "${python_three:-false}" == "true" ]]; then + # To add Py37 support to the Pex, we will need to ensure we have Py37 wheels built and then add "cp-37-m" here. + abis=("cp-36-m") + fi for platform in "${linux_platform_noabi}" "${osx_platform_noabi}"; do - for abi in "cp-27-mu" "cp-27-m"; do + for abi in "${abis[@]}"; do platforms=("${platforms[@]}" "${platform}-${abi}") done done - local dest="${ROOT}/dist/pants.${PANTS_UNSTABLE_VERSION}.pex" - local stable_dest="${DEPLOY_DIR}/pex/pants.${PANTS_STABLE_VERSION}.pex" + local dest="${ROOT}/dist/pants.${PANTS_UNSTABLE_VERSION}.${dest_suffix}" + local stable_dest="${DEPLOY_DIR}/pex/pants.${PANTS_STABLE_VERSION}.${dest_suffix}" ;; *) echo >&2 "Bad build_pex mode ${mode}" diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache index d58b04c4e5e..ca123a5d3b6 100644 --- a/build-support/travis/travis.yml.mustache +++ b/build-support/travis/travis.yml.mustache @@ -656,17 +656,19 @@ py36_jvm_tests: &py36_jvm_tests # Deploy # ------------------------------------------------------------------------- -deploy_stable_multiplatform_pex: &deploy_stable_multiplatform_pex - name: "Deploy stable multiplatform pants.pex" +base_deploy: &base_deploy os: linux language: python + env: + - &base_deploy_env RUN_PANTS_FROM_PEX=1 + +base_deploy_stable_muliplatform_pex: &base_deploy_stable_muliplatform_pex + <<: *base_deploy stage: *build_stable env: - - CACHE_NAME=linuxpexdeploystable - - RUN_PANTS_FROM_PEX=1 - - PANTS_PEX_RELEASE=stable + - &base_deploy_stable_env PANTS_PEX_RELEASE=stable script: - - ./build-support/bin/release.sh -p + - ./build-support/bin/release.sh -p ${RELEASE_ARGS} deploy: # See https://docs.travis-ci.com/user/deployment/releases/ provider: releases @@ -681,17 +683,55 @@ deploy_stable_multiplatform_pex: &deploy_stable_multiplatform_pex tags: true repo: pantsbuild/pants -deploy_unstable_multiplatform_pex: &deploy_unstable_multiplatform_pex - name: "Deploy unstable multiplatform pants.pex" - os: linux - language: python +py27_deploy_stable_multiplatform_pex: &py27_deploy_stable_multiplatform_pex + <<: *base_deploy_stable_muliplatform_pex + name: "Deploy stable multiplatform pants.pex (Py2.7 PEX)" + python: 2.7 + env: + - *base_deploy_env + - *base_deploy_stable_env + - RELEASE_ARGS="" + - CACHE_NAME=linuxpexdeploystable.py27 + +py36_deploy_stable_multiplatform_pex: &py36_deploy_stable_multiplatform_pex + <<: *base_deploy_stable_muliplatform_pex + name: "Deploy stable multiplatform pants.pex (Py3.6 PEX)" + python: 3.6 + env: + - *base_deploy_env + - *base_deploy_stable_env + - RELEASE_ARGS="-3" + - CACHE_NAME=linuxpexdeploystable.py36 + +base_deploy_unstable_multiplatform_pex: &base_deploy_unstable_multiplatform_pex + <<: *base_deploy_unstable_multiplatform_pex stage: *build_unstable env: - - CACHE_NAME=linuxpexdeployunstable - - RUN_PANTS_FROM_PEX=1 - - PREPARE_DEPLOY=1 + - &base_deploy_unstable_env PREPARE_DEPLOY=1 script: - - ./build-support/bin/release.sh -p && mkdir -p dist/deploy/pex/ && mv dist/pants*.pex dist/deploy/pex/ + - ./build-support/bin/release.sh -p ${RELEASE_ARGS} + - mkdir -p dist/deploy/pex/ + - mv dist/pants*.pex dist/deploy/pex/ + +py27_deploy_unstable_multiplatform_pex: &py27_deploy_unstable_multiplatform_pex + <<: *base_deploy_unstable_multiplatform_pex + name: "Deploy unstable multiplatform pants.pex (Py2.7 PEX)" + python: 2.7 + env: + - *base_deploy_env + - *base_deploy_unstable_env + - RELEASE_ARGS="" + - CACHE_NAME=linuxpexdeployunstable.py27 + +py36_deploy_unstable_multiplatform_pex: &py36_deploy_unstable_multiplatform_pex + <<: *base_deploy_unstable_multiplatform_pex + name: "Deploy unstable multiplatform pants.pex (Py3.6 PEX)" + python: 3.6 + env: + - *base_deploy_env + - *base_deploy_unstable_env + - RELEASE_ARGS="-3" + - CACHE_NAME=linuxpexdeployunstable.py36 # ------------------------------------------------------------------------- # Test matrix @@ -805,8 +845,11 @@ matrix: - <<: *py27_jvm_tests - <<: *py36_jvm_tests - - <<: *deploy_stable_multiplatform_pex - - <<: *deploy_unstable_multiplatform_pex + - <<: *py27_deploy_stable_multiplatform_pex + - <<: *py36_deploy_stable_multiplatform_pex + + - <<: *py27_deploy_unstable_multiplatform_pex + - <<: *py36_deploy_unstable_multiplatform_pex deploy: # Deploy whatever a previous stage has left in dist/deploy.