Upgrade to Pants 1.14.0 (#10)

Update ci config and pants script.

Author: Benjy Weinberger

Diff for: .github/workflows/pants.yaml

@@ -18,17 +18,34 @@ jobs:
   build:
     name: Perform CI Checks
     needs: org-check
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        # The python version to run Pants itself on.
+        python-version: [3.9]
     steps:
-    - uses: actions/checkout@v2
-    - uses: actions/cache@v2
-      id: cache
+    - uses: actions/checkout@v3
+    - uses: pantsbuild/actions/init-pants@v2
+      # This action bootstraps pants and manages 2-3 GHA caches.
+      # See: github.com/pantsbuild/actions/tree/main/init-pants/
       with:
-        path: |
-          ~/.cache/pants/setup
-          ~/.cache/pants/lmdb_store
-          ~/.cache/pants/named_caches
-        key: ${{ runner.os }}-
+        pants-python-version: ${{ matrix.python-version }}
+        # cache0 makes it easy to bust the cache if needed
+        # just increase the integer to start with a fresh cache
+        gha-cache-key: cache0-py${{ matrix.python-version }}
+        # The Kotlin backend uses named_caches for Coursier state,
+        # so it is appropriate to invalidate on lockfile changes.
+        named-caches-hash: ${{ hashFiles('3rdparty/jvm/default.lock') }}
+        # If you're not using a fine-grained remote caching service (see https://www.pantsbuild.org/docs/remote-caching),
+        # then you may also want to preserve the local Pants cache (lmdb_store). However this must invalidate for
+        # changes to any file that can affect the build, so may not be practical in larger repos.
+        # A remote cache service integrates with Pants's fine-grained invalidation and avoids these problems.
+        cache-lmdb-store: 'true'  # defaults to 'false'
+        # Note that named_caches and lmdb_store falls back to partial restore keys which
+        # may give a useful partial result that will save time over completely clean state,
+        # but will cause the cache entry to grow without bound over time.
+        # See https://pants.readme.io/docs/using-pants-in-ci for tips on how to periodically clean it up.
+        # Alternatively you change gha-cache-key to ignore old caches.
     - name: Bootstrap Pants
       run: ./pants --version
     - name: Check Pants config files
@@ -39,7 +56,7 @@ jobs:
       run: |
         ./pants package ::
     - name: Upload Pants log
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v3
       with:
         name: pants-log
         path: .pants.d/pants.log

Diff for: pants

@@ -12,10 +12,24 @@
 
 set -eou pipefail
 
+# an arbitrary number: bump when there's a change that someone might want to query for
+# (e.g. checking $(PANTS_BOOTSTRAP_TOOLS=1 ./pants version) >= ...)
+SCRIPT_VERSION=1
+
+# Source any custom bootstrap settings for Pants from PANTS_BOOTSTRAP if it exists.
+: ${PANTS_BOOTSTRAP:=".pants.bootstrap"}
+if [[ -f "${PANTS_BOOTSTRAP}" ]]; then
+  source "${PANTS_BOOTSTRAP}"
+fi
+
 # NOTE: To use an unreleased version of Pants from the pantsbuild/pants main branch,
 #  locate the main branch SHA, set PANTS_SHA=<SHA> in the environment, and run this script as usual.
 #
 # E.g., PANTS_SHA=725fdaf504237190f6787dda3d72c39010a4c574 ./pants --version
+#
+# You can also use PANTS_VERSION=<VERSION> to override the config version that is in the pants.toml file.
+#
+# E.g., PANTS_VERSION=2.13.0 ./pants --version
 
 PYTHON_BIN_NAME="${PYTHON:-unspecified}"
 
@@ -34,9 +48,9 @@ fi
 
 PANTS_BOOTSTRAP="${PANTS_SETUP_CACHE}/bootstrap-$(uname -s)-$(uname -m)"
 
-_PEX_VERSION=2.1.42
+_PEX_VERSION=2.1.103
 _PEX_URL="https://github.com/pantsbuild/pex/releases/download/v${_PEX_VERSION}/pex"
-_PEX_EXPECTED_SHA256="69d6b1b1009b00dd14a3a9f19b72cff818a713ca44b3186c9b12074b2a31e51f"
+_PEX_EXPECTED_SHA256="4d45336511484100ae4e2bab24542a8b86b12c8cb89230463593c60d08c4b8d3"
 
 VIRTUALENV_VERSION=20.4.7
 VIRTUALENV_REQUIREMENTS=$(
@@ -58,6 +72,8 @@ COLOR_GREEN="\x1b[32m"
 COLOR_YELLOW="\x1b[33m"
 COLOR_RESET="\x1b[0m"
 
+INSTALL_URL="https://www.pantsbuild.org/docs/installation"
+
 function log() {
   echo -e "$@" 1>&2
 }
@@ -87,13 +103,18 @@ function get_exe_path_or_die {
   fi
 }
 
-function get_pants_config_value {
+function get_pants_config_string_value {
   local config_key="$1"
   local optional_space="[[:space:]]*"
   local prefix="^${config_key}${optional_space}=${optional_space}"
   local raw_value
-  raw_value="$(sed -ne "/${prefix}/ s#${prefix}##p" "${PANTS_TOML}")"
-  echo "${raw_value}"  | tr -d \"\' && return 0
+  raw_value="$(sed -ne "/${prefix}/ s|${prefix}||p" "${PANTS_TOML}")"
+  local optional_suffix="${optional_space}(#.*)?$"
+  echo "${raw_value}" \
+    | sed -E \
+      -e "s|^'([^']*)'${optional_suffix}|\1|" \
+      -e 's|^"([^"]*)"'"${optional_suffix}"'$|\1|' \
+    && return 0
   return 0
 }
 
@@ -125,11 +146,16 @@ function determine_pants_version {
     return
   fi
 
-  pants_version="$(get_pants_config_value 'pants_version')"
+  if [ -n "${PANTS_VERSION:-}" ]; then
+	echo "${PANTS_VERSION}"
+    return
+  fi
+
+  pants_version="$(get_pants_config_string_value 'pants_version')"
   if [[ -z "${pants_version}" ]]; then
     die "Please explicitly specify the \`pants_version\` in your \`pants.toml\` under the \`[GLOBAL]\` scope.
 See https://pypi.org/project/pantsbuild.pants/#history for all released versions
-and https://www.pantsbuild.org/docs/installation for more instructions."
+and ${INSTALL_URL} for more instructions."
   fi
   pants_major_version="$(echo "${pants_version}" | cut -d '.' -f1)"
   pants_minor_version="$(echo "${pants_version}" | cut -d '.' -f2)"
@@ -191,8 +217,8 @@ function determine_default_python_exe {
     if [[ -z "${interpreter_path}" ]]; then
       continue
     fi
-    # Check if the Python version is installed via Pyenv but not activated.
-    if [[ "$("${interpreter_path}" --version 2>&1 > /dev/null)" == "pyenv: python${version}"* ]]; then
+    # Check if a version is shimmed by pyenv or asdf but not configured.
+    if ! "${interpreter_path}" --version >/dev/null 2>&1; then
       continue
     fi
     if [[ -n "$(check_python_exe_compatible_version "${interpreter_path}")" ]]; then
@@ -248,9 +274,13 @@ function bootstrap_pex {
       mkdir -p "${PANTS_BOOTSTRAP}"
       local staging_dir
       staging_dir=$(tempdir "${PANTS_BOOTSTRAP}")
-      cd "${staging_dir}"
-      curl -LO "${_PEX_URL}"
-      fingerprint="$(compute_sha256 "${python}" "pex")"
+      curl --proto "=https" \
+           --tlsv1.2 \
+           --silent \
+           --location \
+           -o "${staging_dir}/pex" \
+           "${_PEX_URL}"
+      fingerprint="$(compute_sha256 "${python}" "${staging_dir}/pex")"
       if [[ "${_PEX_EXPECTED_SHA256}" != "${fingerprint}" ]]; then
         die "SHA256 of ${_PEX_URL} is not as expected. Aborting."
       fi
@@ -263,12 +293,22 @@ function bootstrap_pex {
   echo "${bootstrapped}"
 }
 
-function scrub_PEX_env_vars {
+function scrub_env_vars {
   # Ensure the virtualenv PEX runs as shrink-wrapped.
   # See: https://github.com/pantsbuild/setup/issues/105
-  if [[ -n "${!PEX_@}" ]]; then
-    warn "Scrubbing ${!PEX_@}"
-    unset "${!PEX_@}"
+  local -r pex_env_vars=(${!PEX_@})
+  if [[ ! ${#pex_env_vars[@]} -eq 0 ]]; then
+    local -r pex_env_vars_to_scrub="${pex_env_vars[@]/PEX_ROOT}"
+    if [[ -n "${pex_env_vars_to_scrub[@]}" ]]; then
+      warn "Scrubbing ${pex_env_vars_to_scrub[@]}"
+      unset ${pex_env_vars_to_scrub[@]}
+    fi
+  fi
+  # Also ensure pip doesn't think packages on PYTHONPATH
+  # are already installed.
+  if [ -n "${PYTHONPATH:-}" ]; then
+    warn "Scrubbing PYTHONPATH"
+    unset PYTHONPATH
   fi
 }
 
@@ -282,11 +322,10 @@ function bootstrap_virtualenv {
       mkdir -p "${PANTS_BOOTSTRAP}"
       local staging_dir
       staging_dir=$(tempdir "${PANTS_BOOTSTRAP}")
-      cd "${staging_dir}"
-      echo "${VIRTUALENV_REQUIREMENTS}" > requirements.txt
+      echo "${VIRTUALENV_REQUIREMENTS}" > "${staging_dir}/requirements.txt"
       (
-        scrub_PEX_env_vars
-        "${python}" "${pex_path}" -r requirements.txt -c virtualenv -o virtualenv.pex
+        scrub_env_vars
+        "${python}" "${pex_path}" -r "${staging_dir}/requirements.txt" -c virtualenv -o "${staging_dir}/virtualenv.pex"
       )
       mkdir -p "$(dirname "${bootstrapped}")"
       mv -f "${staging_dir}/virtualenv.pex" "${bootstrapped}"
@@ -307,7 +346,12 @@ function get_version_for_sha {
 
   # Retrieve the Pants version associated with this commit.
   local pants_version
-  pants_version="$(curl --fail -sL "https://raw.githubusercontent.com/pantsbuild/pants/${sha}/src/python/pants/VERSION")"
+  pants_version="$(curl --proto "=https" \
+                        --tlsv1.2 \
+                        --fail \
+                        --silent \
+                        --location \
+                        "https://raw.githubusercontent.com/pantsbuild/pants/${sha}/src/python/pants/VERSION")"
 
   # Construct the version as the release version from src/python/pants/VERSION, plus the string `+gitXXXXXXXX`,
   # where the XXXXXXXX is the first 8 characters of the SHA.
@@ -318,17 +362,27 @@ function bootstrap_pants {
   local pants_version="$1"
   local python="$2"
   local pants_sha="${3:-}"
+  local pants_debug="${4:-}"
 
-  local pants_requirement="pantsbuild.pants==${pants_version}"
+  local pants_requirements=(pantsbuild.pants==${pants_version})
   local maybe_find_links
   if [[ -z "${pants_sha}" ]]; then
     maybe_find_links=""
   else
     maybe_find_links="--find-links=$(find_links_url "${pants_version}" "${pants_sha}")"
-   fi
+  fi
+
+  local debug_suffix
+  if [[ -z "${pants_debug}" ]]; then
+    debug_suffix=""
+  else
+    debug_suffix="-debug"
+    pants_requirements+=(debugpy==1.6.0)
+  fi
+
   local python_major_minor_version
   python_major_minor_version="$(get_python_major_minor_version "${python}")"
-  local target_folder_name="${pants_version}_py${python_major_minor_version}"
+  local target_folder_name="${pants_version}_py${python_major_minor_version}${debug_suffix}"
   local bootstrapped="${PANTS_BOOTSTRAP}/${target_folder_name}"
 
   if [[ ! -d "${bootstrapped}" ]]; then
@@ -338,13 +392,15 @@ function bootstrap_pants {
       staging_dir=$(tempdir "${PANTS_BOOTSTRAP}")
       local virtualenv_path
       virtualenv_path="$(bootstrap_virtualenv "${python}")" || exit 1
-      green "Installing ${pants_requirement} into a virtual environment at ${bootstrapped}"
+      green "Installing ${pants_requirements[@]} into a virtual environment at ${bootstrapped}"
       (
-        scrub_PEX_env_vars
+        scrub_env_vars
         # shellcheck disable=SC2086
-        "${python}" "${virtualenv_path}" --no-download "${staging_dir}/install" && \
-        "${staging_dir}/install/bin/pip" install -U pip && \
-        "${staging_dir}/install/bin/pip" install ${maybe_find_links} --progress-bar off "${pants_requirement}"
+        "${python}" "${virtualenv_path}" --quiet --no-download "${staging_dir}/install" && \
+        # Grab the latest pip, but don't advance setuptools past 58 which drops support for the
+        # `setup` kwarg `use_2to3` which Pants 1.x sdist dependencies (pystache) use.
+        "${staging_dir}/install/bin/pip" install --quiet -U pip "setuptools<58" && \
+        "${staging_dir}/install/bin/pip" install ${maybe_find_links} --quiet --progress-bar off "${pants_requirements[@]}"
       ) && \
       ln -s "${staging_dir}/install" "${staging_dir}/${target_folder_name}" && \
       mv "${staging_dir}/${target_folder_name}" "${bootstrapped}" && \
@@ -354,19 +410,101 @@ function bootstrap_pants {
   echo "${bootstrapped}"
 }
 
+function run_bootstrap_tools {
+  # functionality for introspecting the bootstrapping process, without actually doing it
+  if [[ "${PANTS_BOOTSTRAP_TOOLS}" -gt "${SCRIPT_VERSION}" ]]; then
+      die "$0 script (bootstrap version ${SCRIPT_VERSION}) is too old for this invocation (with PANTS_BOOTSTRAP_TOOLS=${PANTS_BOOTSTRAP_TOOLS}).
+Please update it by following ${INSTALL_URL}"
+  fi
+
+  case "${1:-}" in
+    bootstrap-cache-key)
+      local pants_version=$(determine_pants_version)
+      local python="$(determine_python_exe "${pants_version}")"
+      # the python above may be a shim (e.g. pyenv or homebrew), so let's get an estimate of the
+      # actual path, as will be symlinked in the virtualenv. (NB. virtualenv does more complicated
+      # things, but we at least emulate the symlink-resolution that it does.)
+      local python_executable_path="$("${python}" -c 'import os, sys; print(os.path.realpath(sys.executable))')"
+
+      local requirements_file="$(mktemp)"
+      echo "${VIRTUALENV_REQUIREMENTS}" > "${requirements_file}"
+      local virtualenv_requirements_sha256="$(compute_sha256 "${python}" "${requirements_file}")"
+      rm "${requirements_file}"
+
+      local parts=(
+        "os_name=$(uname -s)"
+        "arch=$(uname -m)"
+        "python_path=${python}"
+        "python_executable_path=${python_executable_path}"
+        # the full interpreter information, for maximum compatibility
+        "python_version=$("$python" --version)"
+        "pex_version=${_PEX_VERSION}"
+        "virtualenv_requirements_sha256=${virtualenv_requirements_sha256}"
+        "pants_version=${pants_version}"
+      )
+      echo "${parts[*]}"
+      ;;
+    bootstrap-version)
+      echo "${SCRIPT_VERSION}"
+      ;;
+    help|"")
+      cat <<EOF
+Usage: PANTS_BOOTSTRAP_TOOLS=1 $0 ...
+
+Subcommands:
+  bootstrap-cache-key
+    Print an opaque that can be used as a key for accurate and safe caching of
+    the pants bootstrap directories.
+
+    (Added in bootstrap version 1.)
+
+  bootstrap-version
+    Print a version number for the bootstrap script itself.
+
+    Distributed scripts (such as reusable CI formulae) that use these bootstrap
+    tools should set PANTS_BOOTSTRAP_TOOLS to the minimum script version for the
+    features they require. For example, if 'some-tool' was added in version 123:
+
+        PANTS_BOOTSTRAP_TOOLS=123 ./pants some-tool
+
+    (Added in bootstrap version 1.)
+EOF
+      ;;
+    *)
+      die "Unknown subcommand for bootstrap tools: $1. Do you mean to run without PANTS_BOOTSTRAP_TOOLS=1? Or, update this script ($INSTALL_URL)?"
+  esac
+}
+
+if [[ "${PANTS_BOOTSTRAP_TOOLS:-}" -gt 0 ]]; then
+    run_bootstrap_tools "$@"
+    exit 0
+fi
+
 # Ensure we operate from the context of the ./pants buildroot.
 cd "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"
 pants_version="$(determine_pants_version)"
 python="$(determine_python_exe "${pants_version}")"
-pants_dir="$(bootstrap_pants "${pants_version}" "${python}" "${PANTS_SHA:-}")" || exit 1
+pants_dir="$(bootstrap_pants "${pants_version}" "${python}" "${PANTS_SHA:-}" "${PANTS_DEBUG:-}")" || exit 1
 
 pants_python="${pants_dir}/bin/python"
-pants_binary="${pants_dir}/bin/pants"
+pants_binary=(${pants_dir}/bin/pants)
 pants_extra_args=""
 if [[ -n "${PANTS_SHA:-}" ]]; then
   pants_extra_args="${pants_extra_args} --python-repos-repos=$(find_links_url "$pants_version" "$PANTS_SHA")"
 fi
 
+pants_debug_args=()
+if [ -n "${PANTS_DEBUG:-}" ]; then
+  if [[ "$*" != *"--no-pantsd"* ]]; then
+    echo "Error! Must pass '--no-pantsd' when using PANTS_DEBUG"
+    exit 1
+  fi
+  # NB: We can't invoke `-m debugpy` as that'll prepend CWD to sys.path which might have unintended side-effects.
+  # `-c` also prepends, but we strip that ourselves.
+  pants_binary=(-c "__import__(\"sys\").path.pop(0);__import__(\"debugpy.server.cli\").server.cli.main()" --listen 127.0.0.1:5678 --wait-for-client "${pants_binary[@]}")
+  echo "Will launch debugpy server at '127.0.0.1:5678' waiting for client connection."
+fi
+
 # shellcheck disable=SC2086
-exec "${pants_python}" "${pants_binary}" ${pants_extra_args} \
+exec "${pants_python}" "${pants_binary[@]}" ${pants_extra_args} \
   --pants-bin-name="${PANTS_BIN_NAME}" --pants-version=${pants_version} "$@"