Pants upgrade to 1.26 (#149)

This upgrades Pants from 1.23 to 1.26 to fix some build issues related to rust_setuptools.
aurora-scheduler · Feb 18, 2021 · acccb61 · acccb61
1 parent 16adce8
commit acccb61
Show file tree

Hide file tree

Showing 2 changed files with 91 additions and 81 deletions.
diff --git a/pants b/pants
@@ -6,35 +6,33 @@
 # This ./pants bootstrap script comes from the pantsbuild/setup
 # project. It is intended to be checked into your code repository so
 # that other developers have the same setup.
-
 #
 # Learn more here: https://www.pantsbuild.org/docs/installation
 # ====================================================================
 
 set -eou pipefail
 
+# NOTE: To use an unreleased version of Pants from the pantsbuild/pants master branch,
+#  locate the master branch SHA, set PANTS_SHA=<SHA> in the environment, and run this script as usual.
+#
+# E.g., PANTS_SHA=725fdaf504237190f6787dda3d72c39010a4c574 ./pants --version
+
 PYTHON_BIN_NAME="${PYTHON:-unspecified}"
 
-# Set one of these to specify a non-standard location for this script to read the pants version from.
+# Set this to specify a non-standard location for this script to read the Pants version from.
 # NB: This will *not* cause Pants itself to use this location as a config file.
 #     You can use PANTS_CONFIG_FILES or --pants-config-files to do so.
-PANTS_INI=${PANTS_INI:-pants.ini}
 PANTS_TOML=${PANTS_TOML:-pants.toml}
 
-# NOTE: To use an unreleased version of Pants from the pantsbuild/pants master branch,
-#  locate the master branch SHA, set PANTS_SHA=<SHA> in the environment, and run this script as usual.
-#
-# E.g., PANTS_SHA=725fdaf504237190f6787dda3d72c39010a4c574 ./pants --version
-
 PANTS_BIN_NAME="${PANTS_BIN_NAME:-$0}"
 
-PANTS_HOME="${PANTS_HOME:-${XDG_CACHE_HOME:-$HOME/.cache}/pants/setup}"
+PANTS_SETUP_CACHE="${PANTS_SETUP_CACHE:-${XDG_CACHE_HOME:-$HOME/.cache}/pants/setup}"
 # If given a relative path, we fix it to be absolute.
-if [[ "$PANTS_HOME" != /* ]]; then
-  PANTS_HOME="${PWD}/${PANTS_HOME}"
+if [[ "$PANTS_SETUP_CACHE" != /* ]]; then
+  PANTS_SETUP_CACHE="${PWD}/${PANTS_SETUP_CACHE}"
 fi
 
-PANTS_BOOTSTRAP="${PANTS_HOME}/bootstrap-$(uname -s)-$(uname -m)"
+PANTS_BOOTSTRAP="${PANTS_SETUP_CACHE}/bootstrap-$(uname -s)-$(uname -m)"
 
 VENV_VERSION=${VENV_VERSION:-16.4.3}
 
@@ -63,38 +61,24 @@ function tempdir {
 }
 
 function get_exe_path_or_die {
-  exe="$1"
+  local exe="$1"
   if ! command -v "${exe}"; then
     die "Could not find ${exe}. Please ensure ${exe} is on your PATH."
   fi
 }
 
-pants_config_file=""
-if [[ -f "${PANTS_INI}" ]]; then
-  pants_config_file="${PANTS_INI}"
-fi
-if [[ -f "${PANTS_TOML}" ]]; then
-  pants_config_file="${PANTS_TOML}"
-fi
-
 function get_pants_config_value {
-  config_key="$1"
-  optional_space="[[:space:]]*"
-  if [[ "${pants_config_file}" == *.ini ]]; then
-    valid_delimiters="[:=]"
-    prefix="^${config_key}${optional_space}${valid_delimiters}${optional_space}"
-    sed -ne "/${prefix}/ s#${prefix}##p" "${PANTS_INI}" && return 0
-  fi
-  if [[ "${pants_config_file}" == *.toml ]]; then
-    prefix="^${config_key}${optional_space}=${optional_space}"
-    raw_value="$(sed -ne "/${prefix}/ s#${prefix}##p" "${PANTS_TOML}")"
-    echo "${raw_value}"  | tr -d \"\' && return 0
-  fi
+  local config_key="$1"
+  local optional_space="[[:space:]]*"
+  local prefix="^${config_key}${optional_space}=${optional_space}"
+  local raw_value
+  raw_value="$(sed -ne "/${prefix}/ s#${prefix}##p" "${PANTS_TOML}")"
+  echo "${raw_value}"  | tr -d \"\' && return 0
   return 0
 }
 
 function get_python_major_minor_version {
-  python_exe="$1"
+  local python_exe="$1"
   "$python_exe" <<EOF
 import sys
 major_minor_version = ''.join(str(version_num) for version_num in sys.version_info[0:2])
@@ -103,44 +87,69 @@ EOF
 }
 
 # The high-level flow:
-# 1.) Resolve the Python interpreter, first reading from the env var $PYTHON,
-#     then defaulting to Python 3.6+.
-# 2.) Resolve the Pants version from config or from the latest stable
-#     release to PyPI, so that we know what to name the venv (virtual environment) folder and what
-#     to install with Pip.
+#
+# 1.) Resolve the Pants version from config so that we know what interpreters we can use, what to name the venv,
+#     and what to install via pip.
+# 2.) Resolve the Python interpreter, first reading from the env var $PYTHON, then using a default based on the Pants
+#     version.
 # 3.) Check if the venv already exists via a naming convention, and create the venv if not found.
 # 4.) Execute Pants with the resolved Python interpreter and venv.
 #
 # After that, Pants itself will handle making sure any requested plugins
 # are installed and up to date.
 
 function determine_pants_version {
-
   if [ -n "${PANTS_SHA:-}" ]; then
     # get_version_for_sha will echo the version, thus "returning" it from this function.
     get_version_for_sha "$PANTS_SHA"
     return
   fi
 
-  python="$1"
-
   pants_version="$(get_pants_config_value 'pants_version')"
   if [[ -z "${pants_version}" ]]; then
-    pants_version="$(curl -sSL https://pypi.python.org/pypi/pantsbuild.pants/json |
-      "${python}" -c "import json, sys; print(json.load(sys.stdin)['info']['version'])")"
+    die "Please explicitly specify the \`pants_version\` in your \`pants.toml\` under the
+\`[GLOBAL]\` scope. See https://pypi.org/project/pantsbuild.pants/#history for all released
+versions and https://www.pantsbuild.org/docs/installation for more instructions."
   fi
   pants_major_version="$(echo "${pants_version}" | cut -d '.' -f1)"
   pants_minor_version="$(echo "${pants_version}" | cut -d '.' -f2)"
-  if [[ "${pants_major_version}" -eq 1 && "${pants_minor_version}" -le 22 ]]; then
-    die "This version of the \`./pants\` script does not work with Pants <= 1.22.0. Instead,
-either upgrade your \`pants_version\` or use the version of the \`./pants\` script at
-https://raw.githubusercontent.com/pantsbuild/setup/d1da382f6de0420940ec6007a39cba87c21075c6/pants."
+  # 1.26 is the first version to support `pants.toml`, so we fail eagerly if using an outdated version.
+  if [[ "${pants_major_version}" -eq 1 && "${pants_minor_version}" -le 25 ]]; then
+    die "This version of the \`./pants\` script does not work with Pants <= 1.25.0 (and it also requires using \`pants.toml\`,
+rather than \`pants.ini\`). Instead, either upgrade your \`pants_version\` or use the version of the \`./pants\` script
+at https://raw.githubusercontent.com/Eric-Arellano/setup/0d445edef57cb89fd830db70810e38f050b0a268/pants."
   fi
   echo "${pants_version}"
 }
 
+function set_supported_python_versions {
+  local pants_version="$1"
+  local pants_major_version
+  local pants_minor_version
+  pants_major_version="$(echo "${pants_version}" | cut -d '.' -f1)"
+  pants_minor_version="$(echo "${pants_version}" | cut -d '.' -f2)"
+  if [[ "${pants_major_version}" -eq 1 ]]; then
+    supported_python_versions_decimal=('3.6' '3.7' '3.8')
+    supported_python_versions_int=('36' '37' '38')
+    supported_message='3.6, 3.7, or 3.8'
+  elif [[ "${pants_major_version}" -eq 2 && "${pants_minor_version}" -eq 0 ]]; then
+    supported_python_versions_decimal=('3.6' '3.7' '3.8')
+    supported_python_versions_int=('36' '37' '38')
+    supported_message='3.6, 3.7, or 3.8'
+  elif [[ "${pants_major_version}" -eq 2 && "${pants_minor_version}" -eq 1 ]]; then
+    supported_python_versions_decimal=('3.7' '3.8' '3.6')
+    supported_python_versions_int=('37' '38' '36')
+    supported_message='3.7, 3.8, or 3.6 (deprecated)'
+  else
+    supported_python_versions_decimal=('3.7' '3.8')
+    supported_python_versions_int=('37' '38')
+    supported_message='3.7 or 3.8'
+  fi
+}
+
 function determine_default_python_exe {
-  for version in '3.6' '3.7' '3.8' '3.9'; do
+  for version in "${supported_python_versions_decimal[@]}"; do
+    local interpreter_path
     interpreter_path="$(command -v "python${version}")"
     if [[ -z "${interpreter_path}" ]]; then
       continue
@@ -154,22 +163,29 @@ function determine_default_python_exe {
 }
 
 function determine_python_exe {
+  local pants_version="$1"
+  set_supported_python_versions "${pants_version}"
+  local requirement_str="For \`pants_version = \"${pants_version}\"\`, Pants requires Python ${supported_message} to run."
+
+  local python_bin_name
   if [[ "${PYTHON_BIN_NAME}" != 'unspecified' ]]; then
     python_bin_name="${PYTHON_BIN_NAME}"
   else
     python_bin_name="$(determine_default_python_exe)"
     if [[ -z "${python_bin_name}" ]]; then
-      die "No valid Python interpreter found. Pants requires Python 3.6+ to run."
+      die "No valid Python interpreter found. ${requirement_str} Please check that a valid interpreter is installed and on your \$PATH."
     fi
   fi
+  local python_exe
   python_exe="$(get_exe_path_or_die "${python_bin_name}")"
+  local major_minor_version
   major_minor_version="$(get_python_major_minor_version "${python_exe}")"
-  for valid_version in '36' '37' '38' '39'; do
+  for valid_version in "${supported_python_versions_int[@]}"; do
     if [[ "${major_minor_version}" == "${valid_version}" ]]; then
       echo "${python_exe}" && return 0
     fi
   done
-  die "Invalid Python interpreter version for ${python_exe}. Pants requires Python 3.6+ to run."
+  die "Invalid Python interpreter version for ${python_exe}. ${requirement_str}"
 }
 
 # TODO(John Sirois): GC race loser tmp dirs leftover from bootstrap_XXX
@@ -179,6 +195,7 @@ function bootstrap_venv {
   if [[ ! -d "${PANTS_BOOTSTRAP}/${VENV_PACKAGE}" ]]; then
     (
       mkdir -p "${PANTS_BOOTSTRAP}"
+      local staging_dir
       staging_dir=$(tempdir "${PANTS_BOOTSTRAP}")
       cd "${staging_dir}"
       curl -LO "https://pypi.io/packages/source/v/virtualenv/${VENV_TARBALL}"
@@ -193,7 +210,6 @@ function bootstrap_venv {
 function find_links_url {
   local pants_version="$1"
   local pants_sha="$2"
-
   echo -n "https://binaries.pantsbuild.org/wheels/pantsbuild.pants/${pants_sha}/${pants_version/+/%2B}/index.html"
 }
 
@@ -210,22 +226,27 @@ function get_version_for_sha {
 }
 
 function bootstrap_pants {
-  pants_version="$1"
-  python="$2"
-  pants_sha="${3:-}"
+  local pants_version="$1"
+  local python="$2"
+  local pants_sha="${3:-}"
 
-  pants_requirement="pantsbuild.pants==${pants_version}"
+  local pants_requirement="pantsbuild.pants==${pants_version}"
+  local maybe_find_links
   if [[ -z "${pants_sha}" ]]; then
     maybe_find_links=""
   else
     maybe_find_links="--find-links=$(find_links_url "${pants_version}" "${pants_sha}")"
    fi
+  local python_major_minor_version
   python_major_minor_version="$(get_python_major_minor_version "${python}")"
+  local target_folder_name
   target_folder_name="${pants_version}_py${python_major_minor_version}"
 
   if [[ ! -d "${PANTS_BOOTSTRAP}/${target_folder_name}" ]]; then
     (
+      local venv_path
       venv_path="$(bootstrap_venv)"
+      local staging_dir
       staging_dir=$(tempdir "${PANTS_BOOTSTRAP}")
       # shellcheck disable=SC2086
       "${python}" "${venv_path}/virtualenv.py" --no-download "${staging_dir}/install" && \
@@ -241,8 +262,8 @@ function bootstrap_pants {
 
 # Ensure we operate from the context of the ./pants buildroot.
 cd "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"
-python="$(determine_python_exe)"
-pants_version="$(determine_pants_version "${python}")"
+pants_version="$(determine_pants_version)"
+python="$(determine_python_exe "${pants_version}")"
 pants_dir="$(bootstrap_pants "${pants_version}" "${python}" "${PANTS_SHA:-}")"
 pants_python="${pants_dir}/bin/python"
 pants_binary="${pants_dir}/bin/pants"
@@ -251,12 +272,6 @@ if [[ -n "${PANTS_SHA:-}" ]]; then
   pants_extra_args="${pants_extra_args} --python-repos-repos=$(find_links_url "$pants_version" "$PANTS_SHA")"
 fi
 
-# We set the env var no_proxy to '*', to work around an issue with urllib using non
-# async-signal-safe syscalls after we fork a process that has already spawned threads.
-#
-# See https://blog.phusion.nl/2017/10/13/why-ruby-app-servers-break-on-macos-high-sierra-and-what-can-be-done-about-it/
-export no_proxy='*'
-
 # shellcheck disable=SC2086
 exec "${pants_python}" "${pants_binary}" ${pants_extra_args} \
   --pants-bin-name="${PANTS_BIN_NAME}" --pants-version=${pants_version} "$@"
diff --git a/pants.ini → pants.toml b/pants.ini → pants.toml
@@ -12,13 +12,13 @@
 # limitations under the License.
 
 [GLOBAL]
-pants_version: 1.23.0
+pants_version = "1.26.0"
 
-plugins: [
+plugins = [
     'pantsbuild.pants.contrib.python.checks==%(pants_version)s',
   ]
 
-backend_packages: [
+backend_packages = [
     # Pants built-ins:
     'pants.backend.codegen.thrift.python',
     'pants.backend.graph_info',
@@ -27,41 +27,36 @@ backend_packages: [
   ]
 
 # allow imports in BUILD files to read .auroraversion
-build_file_imports: allow
+build_file_imports = "allow"
 
 
 [thrift]
-version: 0.10.0
+version = "0.10.0"
 
 
 [python-setup]
-interpreter_constraints: >3
+interpreter_constraints = ">3"
 
 
 [test.pytest]
 # AURORA-1972: As a workaround for spuriously failing tests, test different
 # targets in isolation
-fast: False
-
-[pytest]
-# FIXME: Workaround for https://github.com/pytest-dev/pytest/issues/4770
-# Can be dropped once we upgrade to pants 1.14.0
-requirements: pytest==3.0.7
+fast = false
 
 # We have some modules that have side-effects upon import, including starting a repl, so we can't
 # use python-eval to validate our BUILD deps currently.
 [lint.python-eval]
-skip: True
+skip = true
 
 
 # We use isort for this.
 [pycheck-import-order]
-skip: True
+skip = true
 
 
 [pycheck-pycodestyle]
 # Code reference is here: http://pep8.readthedocs.org/en/latest/intro.html#error-codes
-ignore: [
+ignore = [
     # Aurora custom ignores:
     'E114',  # indentation is not a multiple of four (comment)
     'E116',  # unexpected indentation (comment)
@@ -93,4 +88,4 @@ ignore: [
 # T800 Instead of Context.CommandError use self.CommandError or cls.CommandError with
 #      instancemethods and classmethods respectively.
 [pycheck-class-factoring]
-skip: True
+skip = true