From f1b710f5b88cee739f044171f9afc4d7c1a2a6d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A1n=20I=2E=20Del=20Valle?= Date: Fri, 21 Aug 2020 14:38:02 -0700 Subject: [PATCH] Update dependencies to Mesos 1.8.1 (#46) ### Description: Upgrading dependencies to Mesos 1.8.1 Upgrading Pants version to 1.23.0 Added Python3.8 to Virtualbox through deadsnakes ppa Bumped up minimum version of Vagrant to 2.2.9 ### Testing Done: Created Vagrant box Did a vagrant up --- RELEASE-NOTES.md | 5 + Vagrantfile | 4 +- build-support/packer/build.sh | 20 +- build.gradle | 4 +- docker-compose.yml | 6 +- examples/vagrant/provision-dev-cluster.sh | 2 +- pants | 283 +++++++++++++++++----- pants.ini | 4 +- 8 files changed, 252 insertions(+), 76 deletions(-) diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 10b23f8d2..3a8864c58 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -1,6 +1,11 @@ 0.24.0 (unreleased) ====== +### New/updated: +- Updated to Mesos dependency to 1.8.1. +- Pants version bumped up to 1.23.0. +- Minimum Vagrant version bumped up to 2.2.9. + 0.23.0 ====== diff --git a/Vagrantfile b/Vagrantfile index c36cb41eb..ffce12edb 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -17,13 +17,13 @@ # Vagrantfile API/syntax version. Don't touch unless you know what you're doing! VAGRANTFILE_API_VERSION = "2" -Vagrant.require_version ">= 2.0.2" +Vagrant.require_version ">= 2.2.9" Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| config.vm.hostname = "aurora.local" # See build-support/packer/README.md for instructions on updating this box. config.vm.box = "apache-aurora/dev-environment" - config.vm.box_version = "0.0.19" + config.vm.box_version = "0.0.20" config.vm.define "devcluster" do |dev| dev.vm.network :private_network, ip: "192.168.33.7", :auto_config => false diff --git a/build-support/packer/build.sh b/build-support/packer/build.sh index 9668f383f..b4f18e8a7 100644 --- a/build-support/packer/build.sh +++ b/build-support/packer/build.sh @@ -17,7 +17,7 @@ set -o errexit set -o nounset set -o verbose -readonly MESOS_VERSION=1.7.2 +readonly MESOS_VERSION=1.8.1 function remove_unused { # The default bento/ubuntu-16.04 image includes juju-core, which adds ~300 MB to our image. @@ -40,10 +40,16 @@ function install_base_packages { libsasl2-dev \ libsvn-dev \ openjdk-8-jdk-headless \ - python-dev + software-properties-common update-alternatives --set java /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java - # Installing zookeeperd as a separate command, as otherwise openjdk-7-jdk is also installed. - apt-get install -y zookeeperd + + # Add repo to control Python version + add-apt-repository -y ppa:deadsnakes/ppa + apt-get update + + # Install python3.8 and also installing zookeeperd as a separate command, + # as otherwise openjdk-7-jdk is also installed. + apt-get install -y python3.8 python3.8-distutils python3.8-dev zookeeperd } function install_docker { @@ -63,7 +69,7 @@ function install_docker { function install_docker2aci { DOCKER2ACI_VERSION="0.17.2" - GOLANG_VERSION="1.14.3" + GOLANG_VERSION="1.14.7" TEMP_PATH=$(mktemp -d) pushd "$TEMP_PATH" @@ -127,10 +133,10 @@ function warm_artifact_cache { # Fetch the mesos egg, needed to build python components. # The mesos.executor target in 3rdparty/python/BUILD expects to find the native egg in # third_party. - SVN_ROOT='https://svn.apache.org/repos/asf/aurora/3rdparty' + MESOS_PY_EGGS='https://dl.bintray.com/aurora-scheduler/python-eggs' pushd "$THIRD_PARTY_DIR" wget -c \ - ${SVN_ROOT}/ubuntu/xenial64/python/mesos.executor-${MESOS_VERSION}-py2.7-linux-x86_64.egg + ${MESOS_PY_EGGS}/ubuntu/xenial64/mesos.executor-${MESOS_VERSION}-py2.7-linux-x86_64.egg popd chown -R vagrant:vagrant aurora diff --git a/build.gradle b/build.gradle index a507dfec7..29c5a6efe 100644 --- a/build.gradle +++ b/build.gradle @@ -215,7 +215,7 @@ project(':api') { task checkPython { doLast { - def python3Executable = ['python3', 'python'].find { python -> + def python3Executable = ['python3', 'python3.8', 'python'].find { python -> try { def check = "import sys; sys.exit(0 if sys.version_info >= (3,8) else 1)" return [python, "-c", check].execute().waitFor() == 0 @@ -391,7 +391,7 @@ dependencies { compile "org.apache.curator:curator-client:${curatorRev}" compile "org.apache.curator:curator-framework:${curatorRev}" compile "org.apache.curator:curator-recipes:${curatorRev}" - compile 'org.apache.mesos:mesos:1.7.2' + compile 'org.apache.mesos:mesos:1.8.1' compile "org.asynchttpclient:async-http-client:${asyncHttpclientRev}" compile "org.apache.shiro:shiro-guice:${shiroRev}" compile "org.apache.shiro:shiro-web:${shiroRev}" diff --git a/docker-compose.yml b/docker-compose.yml index 9790c6f1c..04062ee79 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,7 +14,7 @@ services: ipv4_address: 192.168.33.2 master: - image: aurorascheduler/mesos-master:1.7.2 + image: aurorascheduler/mesos-master:1.8.1 restart: on-failure ports: - "5050:5050" @@ -32,7 +32,7 @@ services: - zk agent: - image: aurorascheduler/mesos-agent:1.7.2 + image: aurorascheduler/mesos-agent:1.8.1 pid: host restart: on-failure ports: @@ -45,6 +45,7 @@ services: MESOS_RESOURCES: ports(*):[11000-11999] MESOS_SYSTEMD_ENABLE_SUPPORT: 'false' MESOS_WORK_DIR: /tmp/mesos + MESOS_ATTRIBUTES: 'zone:east networks: aurora_cluster: ipv4_address: 192.168.33.5 @@ -67,6 +68,7 @@ services: MESOS_MASTER: "zk://192.168.33.2:2181/mesos" EXTRA_SCHEDULER_ARGS: > -http_authentication_mechanism=NONE + -min_required_instances_for_sla_check=1 volumes: - ./:/home/vagrant/aurora networks: diff --git a/examples/vagrant/provision-dev-cluster.sh b/examples/vagrant/provision-dev-cluster.sh index 89c716c90..8353e8b13 100755 --- a/examples/vagrant/provision-dev-cluster.sh +++ b/examples/vagrant/provision-dev-cluster.sh @@ -110,4 +110,4 @@ install_rsyslog_config start_services configure_netrc docker_setup -su vagrant -c "aurorabuild all" +su vagrant -c "aurorabuild scheduler" diff --git a/pants b/pants index 57b4d051d..00c0843e5 100755 --- a/pants +++ b/pants @@ -1,99 +1,262 @@ #!/usr/bin/env bash -# -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Copyright 2020 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). # =============================== NOTE =============================== -# This pants bootstrap script comes from the pantsbuild/setup -# project and is intended to be checked into your code repository so -# that any developer can check out your code and be building it with -# pants with no prior setup needed. +# This ./pants bootstrap script comes from the pantsbuild/setup +# project. It is intended to be checked into your code repository so +# that other developers have the same setup. + # -# You can learn more here: https://pantsbuild.github.io/setup +# Learn more here: https://www.pantsbuild.org/docs/installation # ==================================================================== -set -e -# AURORA-1717: If the PYTHONPATH is tainted, calling `./pants` may fail -# with `ImportError: No module named pants.bin.pants_exe` -unset PYTHONPATH +set -eou pipefail + +PYTHON_BIN_NAME="${PYTHON:-unspecified}" -PYTHON=${PYTHON:-$(which python2.7)} +# Set one of these to specify a non-standard location for this script to read the pants version from. +# NB: This will *not* cause Pants itself to use this location as a config file. +# You can use PANTS_CONFIG_FILES or --pants-config-files to do so. +PANTS_INI=${PANTS_INI:-pants.ini} +PANTS_TOML=${PANTS_TOML:-pants.toml} + +# NOTE: To use an unreleased version of Pants from the pantsbuild/pants master branch, +# locate the master branch SHA, set PANTS_SHA= in the environment, and run this script as usual. +# +# E.g., PANTS_SHA=725fdaf504237190f6787dda3d72c39010a4c574 ./pants --version + +PANTS_BIN_NAME="${PANTS_BIN_NAME:-$0}" + +PANTS_HOME="${PANTS_HOME:-${XDG_CACHE_HOME:-$HOME/.cache}/pants/setup}" +# If given a relative path, we fix it to be absolute. +if [[ "$PANTS_HOME" != /* ]]; then + PANTS_HOME="${PWD}/${PANTS_HOME}" +fi -PANTS_HOME="${PANTS_HOME:-${HOME}/.cache/pants/setup}" PANTS_BOOTSTRAP="${PANTS_HOME}/bootstrap-$(uname -s)-$(uname -m)" -VENV_VERSION=16.6.1 +VENV_VERSION=${VENV_VERSION:-16.4.3} VENV_PACKAGE=virtualenv-${VENV_VERSION} VENV_TARBALL=${VENV_PACKAGE}.tar.gz +COLOR_RED="\x1b[31m" +COLOR_GREEN="\x1b[32m" +COLOR_RESET="\x1b[0m" + +function log() { + echo -e "$@" 1>&2 +} + +function die() { + (($# > 0)) && log "${COLOR_RED}$*${COLOR_RESET}" + exit 1 +} + +function green() { + (($# > 0)) && log "${COLOR_GREEN}$*${COLOR_RESET}" +} + +function tempdir { + mktemp -d "$1"/pants.XXXXXX +} + +function get_exe_path_or_die { + exe="$1" + if ! command -v "${exe}"; then + die "Could not find ${exe}. Please ensure ${exe} is on your PATH." + fi +} + +pants_config_file="" +if [[ -f "${PANTS_INI}" ]]; then + pants_config_file="${PANTS_INI}" +fi +if [[ -f "${PANTS_TOML}" ]]; then + pants_config_file="${PANTS_TOML}" +fi + +function get_pants_config_value { + config_key="$1" + optional_space="[[:space:]]*" + if [[ "${pants_config_file}" == *.ini ]]; then + valid_delimiters="[:=]" + prefix="^${config_key}${optional_space}${valid_delimiters}${optional_space}" + sed -ne "/${prefix}/ s#${prefix}##p" "${PANTS_INI}" && return 0 + fi + if [[ "${pants_config_file}" == *.toml ]]; then + prefix="^${config_key}${optional_space}=${optional_space}" + raw_value="$(sed -ne "/${prefix}/ s#${prefix}##p" "${PANTS_TOML}")" + echo "${raw_value}" | tr -d \"\' && return 0 + fi + return 0 +} + +function get_python_major_minor_version { + python_exe="$1" + "$python_exe" <&1 > /dev/null)" == "pyenv: python${version}"* ]]; then + continue + fi + echo "${interpreter_path}" && return 0 + done +} + +function determine_python_exe { + if [[ "${PYTHON_BIN_NAME}" != 'unspecified' ]]; then + python_bin_name="${PYTHON_BIN_NAME}" + else + python_bin_name="$(determine_default_python_exe)" + if [[ -z "${python_bin_name}" ]]; then + die "No valid Python interpreter found. Pants requires Python 3.6+ to run." + fi + fi + python_exe="$(get_exe_path_or_die "${python_bin_name}")" + major_minor_version="$(get_python_major_minor_version "${python_exe}")" + for valid_version in '36' '37' '38' '39'; do + if [[ "${major_minor_version}" == "${valid_version}" ]]; then + echo "${python_exe}" && return 0 + fi + done + die "Invalid Python interpreter version for ${python_exe}. Pants requires Python 3.6+ to run." } # TODO(John Sirois): GC race loser tmp dirs leftover from bootstrap_XXX # functions. Any tmp dir w/o a symlink pointing to it can go. function bootstrap_venv { - if [[ ! -d "${PANTS_BOOTSTRAP}/${VENV_PACKAGE}" ]] - then + if [[ ! -d "${PANTS_BOOTSTRAP}/${VENV_PACKAGE}" ]]; then ( - mkdir -p "${PANTS_BOOTSTRAP}" && \ - staging_dir=$(tempdir "${PANTS_BOOTSTRAP}") && \ - cd "${staging_dir}" && \ - curl -LO https://pypi.io/packages/source/v/virtualenv/${VENV_TARBALL} && \ - tar -xzf ${VENV_TARBALL} && \ - ln -s "${staging_dir}/${VENV_PACKAGE}" "${staging_dir}/latest" && \ + mkdir -p "${PANTS_BOOTSTRAP}" + staging_dir=$(tempdir "${PANTS_BOOTSTRAP}") + cd "${staging_dir}" + curl -LO "https://pypi.io/packages/source/v/virtualenv/${VENV_TARBALL}" + tar -xzf "${VENV_TARBALL}" + ln -s "${staging_dir}/${VENV_PACKAGE}" "${staging_dir}/latest" mv "${staging_dir}/latest" "${PANTS_BOOTSTRAP}/${VENV_PACKAGE}" ) 1>&2 fi echo "${PANTS_BOOTSTRAP}/${VENV_PACKAGE}" } +function find_links_url { + local pants_version="$1" + local pants_sha="$2" + + echo -n "https://binaries.pantsbuild.org/wheels/pantsbuild.pants/${pants_sha}/${pants_version/+/%2B}/index.html" +} + +function get_version_for_sha { + local sha="$1" + + # Retrieve the Pants version associated with this commit. + local pants_version + pants_version="$(curl --fail -sL "https://raw.githubusercontent.com/pantsbuild/pants/${sha}/src/python/pants/VERSION")" + + # Construct the version as the release version from src/python/pants/VERSION, plus the string `+gitXXXXXXXX`, + # where the XXXXXXXX is the first 8 characters of the SHA. + echo "${pants_version}+git${sha:0:8}" +} + function bootstrap_pants { - pants_requirement="pantsbuild.pants" - pants_version=$( - grep -E "^[[:space:]]*pants_version" pants.ini 2>/dev/null | \ - cut -f2 -d: | tr -d " " - ) - if [[ -n "${pants_version}" ]] - then - pants_requirement="${pants_requirement}==${pants_version}" + pants_version="$1" + python="$2" + pants_sha="${3:-}" + + pants_requirement="pantsbuild.pants==${pants_version}" + if [[ -z "${pants_sha}" ]]; then + maybe_find_links="" else - pants_version="unspecified" - fi + maybe_find_links="--find-links=$(find_links_url "${pants_version}" "${pants_sha}")" + fi + python_major_minor_version="$(get_python_major_minor_version "${python}")" + target_folder_name="${pants_version}_py${python_major_minor_version}" - if [[ ! -d "${PANTS_BOOTSTRAP}/${pants_version}" ]] - then + if [[ ! -d "${PANTS_BOOTSTRAP}/${target_folder_name}" ]]; then ( - venv_path="$(bootstrap_venv)" && \ - staging_dir=$(tempdir "${PANTS_BOOTSTRAP}") && \ - "${PYTHON}" "${venv_path}/virtualenv.py" --no-download "${staging_dir}/install" && \ - "${staging_dir}/install/bin/python" "${staging_dir}/install/bin/pip" install "${pants_requirement}" && \ - ln -s "${staging_dir}/install" "${staging_dir}/${pants_version}" && \ - mv "${staging_dir}/${pants_version}" "${PANTS_BOOTSTRAP}/${pants_version}" + venv_path="$(bootstrap_venv)" + staging_dir=$(tempdir "${PANTS_BOOTSTRAP}") + # shellcheck disable=SC2086 + "${python}" "${venv_path}/virtualenv.py" --no-download "${staging_dir}/install" && \ + "${staging_dir}/install/bin/pip" install -U pip && \ + "${staging_dir}/install/bin/pip" install ${maybe_find_links} --progress-bar off "${pants_requirement}" && \ + ln -s "${staging_dir}/install" "${staging_dir}/${target_folder_name}" && \ + mv "${staging_dir}/${target_folder_name}" "${PANTS_BOOTSTRAP}/${target_folder_name}" && \ + green "New virtual environment successfully created at ${PANTS_BOOTSTRAP}/${target_folder_name}." ) 1>&2 fi - echo "${PANTS_BOOTSTRAP}/${pants_version}" + echo "${PANTS_BOOTSTRAP}/${target_folder_name}" } -pants_dir=$(bootstrap_pants) && \ -exec "${pants_dir}/bin/python" "${pants_dir}/bin/pants" "$@" +# Ensure we operate from the context of the ./pants buildroot. +cd "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +python="$(determine_python_exe)" +pants_version="$(determine_pants_version "${python}")" +pants_dir="$(bootstrap_pants "${pants_version}" "${python}" "${PANTS_SHA:-}")" +pants_python="${pants_dir}/bin/python" +pants_binary="${pants_dir}/bin/pants" +pants_extra_args="" +if [[ -n "${PANTS_SHA:-}" ]]; then + pants_extra_args="${pants_extra_args} --python-repos-repos=$(find_links_url "$pants_version" "$PANTS_SHA")" +fi + +# We set the env var no_proxy to '*', to work around an issue with urllib using non +# async-signal-safe syscalls after we fork a process that has already spawned threads. +# +# See https://blog.phusion.nl/2017/10/13/why-ruby-app-servers-break-on-macos-high-sierra-and-what-can-be-done-about-it/ +export no_proxy='*' + +# shellcheck disable=SC2086 +exec "${pants_python}" "${pants_binary}" ${pants_extra_args} \ + --pants-bin-name="${PANTS_BIN_NAME}" --pants-version=${pants_version} "$@" diff --git a/pants.ini b/pants.ini index 6bd56d7ab..c3c7b5825 100644 --- a/pants.ini +++ b/pants.ini @@ -12,7 +12,7 @@ # limitations under the License. [GLOBAL] -pants_version: 1.15.0 +pants_version: 1.23.0 plugins: [ 'pantsbuild.pants.contrib.python.checks==%(pants_version)s', @@ -35,7 +35,7 @@ version: 0.10.0 [python-setup] -interpreter_constraints: CPython>=2.7,<3 +interpreter_constraints: >3 [test.pytest]