Upgrade pants to 1.26.0

api/src/main/thrift/org/apache/aurora/gen/BUILD

@@ -26,7 +26,7 @@ python_thrift_library(
 
 python_thrift_library(
   name = '_test',
-  sources = globs('*test.thrift'),
+  sources = ['*test.thrift'],
   dependencies = [
     '3rdparty/python:thrift',
   ],

pants

@@ -19,81 +19,200 @@
 # that any developer can check out your code and be building it with
 # pants with no prior setup needed.
 #
-# You can learn more here: https://pantsbuild.github.io/setup
+# You can learn more here: https://www.pantsbuild.org/install.html
 # ====================================================================
-set -e
 
 # AURORA-1717: If the PYTHONPATH is tainted, calling `./pants` may fail
 # with `ImportError: No module named pants.bin.pants_exe`
 unset PYTHONPATH
+set -eou pipefail
 
-PYTHON=${PYTHON:-$(which python2.7)}
+PANTS_INI=${PANTS_INI:-pants.ini}
+PANTS_TOML=${PANTS_TOML:-pants.toml}
+PYTHON_BIN_NAME="${PYTHON:-unspecified}"
 
-PANTS_HOME="${PANTS_HOME:-${HOME}/.cache/pants/setup}"
+PANTS_HOME="${PANTS_HOME:-${XDG_CACHE_HOME:-$HOME/.cache}/pants/setup}"
 PANTS_BOOTSTRAP="${PANTS_HOME}/bootstrap-$(uname -s)-$(uname -m)"
 
-VENV_VERSION=16.6.1
+VENV_VERSION=${VENV_VERSION:-16.4.3}
 
 VENV_PACKAGE=virtualenv-${VENV_VERSION}
 VENV_TARBALL=${VENV_PACKAGE}.tar.gz
 
+COLOR_RED="\x1b[31m"
+COLOR_GREEN="\x1b[32m"
+COLOR_RESET="\x1b[0m"
+
+function log() {
+  echo -e "$@" 1>&2
+}
+
+function die() {
+  (($# > 0)) && log "${COLOR_RED}$*${COLOR_RESET}"
+  exit 1
+}
+
+function green() {
+  (($# > 0)) && log "${COLOR_GREEN}$*${COLOR_RESET}"
+}
+
+function tempdir {
+  mktemp -d "$1"/pants.XXXXXX
+}
+
+function get_exe_path_or_die {
+  exe="$1"
+  if ! command -v "${exe}"; then
+    die "Could not find ${exe}. Please ensure ${exe} is on your PATH."
+  fi
+}
+
+pants_config_file=""
+if [[ -f "${PANTS_INI}" ]]; then
+  pants_config_file="${PANTS_INI}"
+fi
+if [[ -f "${PANTS_TOML}" ]]; then
+  pants_config_file="${PANTS_TOML}"
+fi
+
+function get_pants_config_value {
+  config_key="$1"
+  optional_space="[[:space:]]*"
+  if [[ "${pants_config_file}" == *.ini ]]; then
+    valid_delimiters="[:=]"
+    prefix="^${config_key}${optional_space}${valid_delimiters}${optional_space}"
+    sed -ne "/${prefix}/ s#${prefix}##p" "${PANTS_INI}" && return 0
+  fi
+  if [[ "${pants_config_file}" == *.toml ]]; then
+    prefix="^${config_key}${optional_space}=${optional_space}"
+    raw_value="$(sed -ne "/${prefix}/ s#${prefix}##p" "${PANTS_TOML}")"
+    echo "${raw_value}"  | tr -d \"\' && return 0
+  fi
+  return 0
+}
+
+function get_python_major_minor_version {
+  python_exe="$1"
+  "$python_exe" <<EOF
+import sys
+major_minor_version = ''.join(str(version_num) for version_num in sys.version_info[0:2])
+print(major_minor_version)
+EOF
+}
+
 # The high-level flow:
-# 1.) Grab pants version from pants.ini or default to latest.
-# 2.) Check for a venv via a naming/path convention and execute if found.
-# 3.) Otherwise create venv and re-exec self.
+# 1.) Resolve the Pants version from pants.ini or from the latest stable
+#     release to PyPI, so that we know what to name the venv (virtual environment) folder and what
+#     to install with Pip.
+# 2.) Resolve the Python interpreter, first reading from the env var $PYTHON,
+#     then defaulting to Python 3.6+.
+# 3.) Check if the venv already exists via a naming convention, and create the venv if not found.
+# 4.) Execute Pants with the resolved Python interpreter and venv.
 #
-# After that pants itself will handle making sure any requested plugins
+# After that, Pants itself will handle making sure any requested plugins
 # are installed and up to date.
 
-function tempdir {
-  mktemp -d "$1"/pants.XXXXXX
+function determine_pants_version {
+  pants_version="$(get_pants_config_value 'pants_version')"
+  if [[ -z "${pants_version}" ]]; then
+    pants_version="$(curl -sSL https://pypi.python.org/pypi/pantsbuild.pants/json |
+      python -c "import json, sys; print(json.load(sys.stdin)['info']['version'])")"
+  fi
+  pants_major_version="$(echo "${pants_version}" | cut -d '.' -f1)"
+  pants_minor_version="$(echo "${pants_version}" | cut -d '.' -f2)"
+  if [[ "${pants_major_version}" -eq 1 && "${pants_minor_version}" -le 16 ]]; then
+    die "This version of the \`./pants\` script does not work with Pants <= 1.16.0. Instead,
+either upgrade your \`pants_version\` or use the version of the \`./pants\` script at
+https://raw.githubusercontent.com/pantsbuild/setup/3bb006e4a792ae83d7059ea76c0372ece7c1069d/pants."
+  fi
+  echo "${pants_version}"
+}
+
+function determine_default_python_exe {
+  for version in '3.6' '3.7' '3.8' '3.9'; do
+    interpreter_path="$(command -v "python${version}")"
+    if [[ -z "${interpreter_path}" ]]; then
+      continue
+    fi
+    # Check if the Python version is installed via Pyenv but not activated.
+    if [[ "$("${interpreter_path}" --version 2>&1 > /dev/null)" == "pyenv: python${version}"* ]]; then
+      continue
+    fi
+    echo "${interpreter_path}" && return 0
+  done
+}
+
+function determine_python_exe {
+  if [[ "${PYTHON_BIN_NAME}" != 'unspecified' ]]; then
+    python_bin_name="${PYTHON_BIN_NAME}"
+  else
+    python_bin_name="$(determine_default_python_exe)"
+    if [[ -z "${python_bin_name}" ]]; then
+      die "No valid Python interpreter found. Pants requires Python 3.6+ to run."
+    fi
+  fi
+  python_exe="$(get_exe_path_or_die "${python_bin_name}")"
+  major_minor_version="$(get_python_major_minor_version "${python_exe}")"
+  for valid_version in '36' '37' '38' '39'; do
+    if [[ "${major_minor_version}" == "${valid_version}" ]]; then
+      echo "${python_exe}" && return 0
+    fi
+  done
+  die "Invalid Python interpreter version for ${python_exe}. Pants requires Python 3.6+ to run."
 }
 
 # TODO(John Sirois): GC race loser tmp dirs leftover from bootstrap_XXX
 # functions.  Any tmp dir w/o a symlink pointing to it can go.
 
 function bootstrap_venv {
-  if [[ ! -d "${PANTS_BOOTSTRAP}/${VENV_PACKAGE}" ]]
-  then
+  if [[ ! -d "${PANTS_BOOTSTRAP}/${VENV_PACKAGE}" ]]; then
     (
-      mkdir -p "${PANTS_BOOTSTRAP}" && \
-      staging_dir=$(tempdir "${PANTS_BOOTSTRAP}") && \
-      cd "${staging_dir}" && \
-      curl -LO https://pypi.io/packages/source/v/virtualenv/${VENV_TARBALL} && \
-      tar -xzf ${VENV_TARBALL} && \
-      ln -s "${staging_dir}/${VENV_PACKAGE}" "${staging_dir}/latest" && \
+      mkdir -p "${PANTS_BOOTSTRAP}"
+      staging_dir=$(tempdir "${PANTS_BOOTSTRAP}")
+      cd "${staging_dir}"
+      curl -LO "https://pypi.io/packages/source/v/virtualenv/${VENV_TARBALL}"
+      tar -xzf "${VENV_TARBALL}"
+      ln -s "${staging_dir}/${VENV_PACKAGE}" "${staging_dir}/latest"
       mv "${staging_dir}/latest" "${PANTS_BOOTSTRAP}/${VENV_PACKAGE}"
     ) 1>&2
   fi
   echo "${PANTS_BOOTSTRAP}/${VENV_PACKAGE}"
 }
 
 function bootstrap_pants {
-  pants_requirement="pantsbuild.pants"
-  pants_version=$(
-    grep -E "^[[:space:]]*pants_version" pants.ini 2>/dev/null | \
-      cut -f2 -d: | tr -d " "
-  )
-  if [[ -n "${pants_version}" ]]
-  then
-    pants_requirement="${pants_requirement}==${pants_version}"
-  else
-    pants_version="unspecified"
-  fi
+  pants_version="$1"
+  python="$2"
+
+  pants_requirement="pantsbuild.pants==${pants_version}"
+  python_major_minor_version="$(get_python_major_minor_version "${python}")"
+  target_folder_name="${pants_version}_py${python_major_minor_version}"
 
-  if [[ ! -d "${PANTS_BOOTSTRAP}/${pants_version}" ]]
-  then
+  if [[ ! -d "${PANTS_BOOTSTRAP}/${target_folder_name}" ]]; then
     (
-      venv_path="$(bootstrap_venv)" && \
-      staging_dir=$(tempdir "${PANTS_BOOTSTRAP}") && \
-      "${PYTHON}" "${venv_path}/virtualenv.py" --no-download "${staging_dir}/install" && \
-      "${staging_dir}/install/bin/python" "${staging_dir}/install/bin/pip" install "${pants_requirement}" && \
-      ln -s "${staging_dir}/install" "${staging_dir}/${pants_version}" && \
-      mv "${staging_dir}/${pants_version}" "${PANTS_BOOTSTRAP}/${pants_version}"
+      venv_path="$(bootstrap_venv)"
+      staging_dir=$(tempdir "${PANTS_BOOTSTRAP}")
+      "${python}" "${venv_path}/virtualenv.py" --no-download "${staging_dir}/install"
+      "${staging_dir}/install/bin/pip" install -U pip
+      "${staging_dir}/install/bin/pip" install "${pants_requirement}"
+      ln -s "${staging_dir}/install" "${staging_dir}/${target_folder_name}"
+      mv "${staging_dir}/${target_folder_name}" "${PANTS_BOOTSTRAP}/${target_folder_name}"
+      green "New virtual environment successfully created at ${PANTS_BOOTSTRAP}/${target_folder_name}."
     ) 1>&2
   fi
-  echo "${PANTS_BOOTSTRAP}/${pants_version}"
+  echo "${PANTS_BOOTSTRAP}/${target_folder_name}"
 }
 
-pants_dir=$(bootstrap_pants) && \
+# Ensure we operate from the context of the ./pants buildroot.
+cd "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"
+pants_version="$(determine_pants_version)"
+python="$(determine_python_exe)"
+pants_dir="$(bootstrap_pants "${pants_version}" "${python}")"
+
+
+# We set the env var no_proxy to '*', to work around an issue with urllib using non
+# async-signal-safe syscalls after we fork a process that has already spawned threads.
+#
+# See https://blog.phusion.nl/2017/10/13/why-ruby-app-servers-break-on-macos-high-sierra-and-what-can-be-done-about-it/
+export no_proxy='*'
+
 exec "${pants_dir}/bin/python" "${pants_dir}/bin/pants" "$@"