Prep support for running Pants with Python 3.10 or 3.11. (#351)

~Look at the Python tag in the Pants PEX filename~ Cycle through all
supported Python versions and check the PEX download URL using HEAD for
each one to infer the Python version to use when installing Pants, when
using a version of Pants that we don't already know which version of
Python to use.

~Drop support for `PANTS_SHA`.~ Done in #376.

~Silence the install step, capturing the output to a `pants-install.log`
file in the science cache.~ Done in #375.

---------

Co-authored-by: Huon Wilson <[email protected]>

Author: kaos

.github/workflows/release.yml

@@ -6,7 +6,15 @@ on:
   workflow_dispatch:
     inputs:
       tag:
-        description: The tag to manually run a deploy for.
+        description: The tag to manually run a deploy for. (example tag `v1.2.3`)
+        required: true
+      prerelease:
+        description: |
+          Is this for a prerelease?
+          (for internal testing, not announced nor published to our brew tap)
+          (example tag `v1.2.3-beta.0`)
+        type: boolean
+        default: false
         required: true
 env:
   CARGO_TERM_COLOR: always
@@ -25,20 +33,24 @@ jobs:
     outputs:
       release-tag: ${{ steps.determine-tag.outputs.release-tag }}
       release-version: ${{ steps.determine-tag.outputs.release-version }}
+      prerelease: ${{ steps.determine-tag.outputs.prerelease }}
     steps:
       - name: Determine Tag
         id: determine-tag
         run: |
           if [[ -n "${{ github.event.inputs.tag }}" ]]; then
             RELEASE_TAG=${{ github.event.inputs.tag }}
+            PRERELEASE=${{ github.event.inputs.prerelease }}
           else
             RELEASE_TAG=${GITHUB_REF#refs/tags/}
+            PRERELEASE=false
           fi
-          if [[ "${RELEASE_TAG}" =~ ^v[0-9]+.[0-9]+.[0-9]+$ ]]; then
+          if [[ "${RELEASE_TAG}" =~ ^v[0-9]+.[0-9]+.[0-9]+$ ]] || [[ "${PRERELEASE}" == true ]]; then
             echo "release-tag=${RELEASE_TAG}" >> $GITHUB_OUTPUT
             echo "release-version=${RELEASE_TAG#v}" >> $GITHUB_OUTPUT
+            echo "prerelease=${PRERELEASE}" >> $GITHUB_OUTPUT
           else
-            echo "::error::Release tag '${RELEASE_TAG}' must match 'v\d+.\d+.\d+'."
+            echo "::error::Release tag '${RELEASE_TAG}' must match 'v\d+.\d+.\d+' when not doing a pre-release."
             exit 1
           fi
   github-release:
@@ -87,7 +99,7 @@ jobs:
           name: scie-pants ${{ needs.determine-tag.outputs.release-version }}
           body_path: ${{ steps.prepare-changelog.outputs.changelog-file }}
           draft: false
-          prerelease: false
+          prerelease: ${{ needs.determine-tag.outputs.prerelease }}
           files: dist/scie-pants-*
           fail_on_unmatched_files: true
           discussion_category_name: Announcements
@@ -113,6 +125,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
       - name: Post Release Announcement to Pants Slack `#announce`
+        if: ${{ needs.determine-tag.outputs.prerelease != 'true' }}
         id: slack
         uses: slackapi/[email protected]
         with:
@@ -148,6 +161,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
       - name: Bump `scie-pants` version in Casks/pants.rb to ${{ needs.determine-tag.outputs.release-tag }}
+        if: ${{ needs.determine-tag.outputs.prerelease != 'true' }}
         uses: actions/github-script@v6
         with:
           github-token: ${{ secrets.TAP_TOKEN }}

CHANGES.md

@@ -1,5 +1,9 @@
 # Release Notes
 
+## 0.12.0
+
+Add support for running Pants with newer versions of Python.
+
 ## 0.11.0
 
 Drop support for `PANTS_SHA` which was deprecated in `0.10.0`.

Cargo.lock

@@ -6,7 +6,7 @@ members = [
 [package]
 name = "scie-pants"
 description = "Protects your Pants from the elements."
-version = "0.11.0"
+version = "0.12.0"
 edition = "2021"
 authors = [
     "John Sirois <[email protected]>",

Cargo.toml

@@ -42,7 +42,7 @@ In the course of development you'll probably only be interested in two invocatio
 The primary goal of the package crate as build system is to support development of the `scie-pants`
 binary with an install of Rust as the ~only requirement (CMake is currently needed as well). This
 necessitates dogfooding the same scie mechanism the final `scie-pants` binary uses in production in
-order to bootstrap a [Python tool chain](pbt.lift.json). This is used build the [`tools.pex`](
+order to bootstrap a [Python tool chain](pbt.toml). This is used to build the [`tools.pex`](
 ../tools/README.md) embedded in the final `scie-pants` for use in all the slow-path / high-logic
 steps like Pants configuration, Pants installation and self-update.
 
@@ -53,9 +53,9 @@ src/main.rs):
 1. A [`ptex` binary](https://github.com/a-scie/ptex) is built via `cargo install`. This bootstraps
    the ability to fetch further requirements.
 2. The current production pins of `ptex` and `scie-jump` are fetched and checksum-verified.
-3. A [`pbt`](pbt.lift.json) scie binary is built to facilitate running Python, Pip and Pex tools.
+3. A [`pbt`](pbt.toml) scie binary is built to facilitate running Python, Pip and Pex tools.
 4. The [tools.pex](../tools) is built.
-5. The `scie-pants` scie binary is built.
+5. The [`scie-pants`](scie-pants.toml) scie binary is built.
 6. The `scie-pants` binary is used to run Pants against the Python tools codebase and then to run a
    series of integration tests exercising the ability to install different Pants vintages and
    configure new Pants projects.

package/README.md

@@ -28,12 +28,28 @@ release = "20240107"
 lazy = true
 version = "3.9.18"
 
+[[lift.interpreters]]
+id = "cpython310"
+provider = "PythonBuildStandalone"
+release = "20240415"
+lazy = true
+version = "3.10.14"
+
+[[lift.interpreters]]
+id = "cpython311"
+provider = "PythonBuildStandalone"
+release = "20240415"
+lazy = true
+version = "3.11.9"
+
 [[lift.interpreter_groups]]
 id = "cpython"
 selector = "{scie.bindings.configure:PYTHON}"
 members = [
     "cpython38",
     "cpython39",
+    "cpython310",
+    "cpython311",
 ]
 
 [[lift.files]]
@@ -183,6 +199,8 @@ args = [
     "{scie.env.PANTS_TOML}",
     "--github-api-bearer-token",
     "{scie.env.PANTS_BOOTSTRAP_GITHUB_API_BEARER_TOKEN}",
+    "--pants-bootstrap-urls",
+    "{scie.env.PANTS_BOOTSTRAP_URLS}",
     "{scie.bindings}",
 ]
 env.remove_re = [
@@ -209,14 +227,14 @@ args = [
     "install-pants",
     "--pants-version",
     "{scie.bindings.configure:PANTS_VERSION}",
+    "--pants-pex-url",
+    "{scie.bindings.configure:PANTS_PEX_URL}",
     "--find-links",
     "{scie.bindings.configure:FIND_LINKS}",
     "--debug",
     "{scie.env.PANTS_DEBUG}",
     "--debugpy-requirement",
     "{scie.env.PANTS_DEBUGPY_VERSION}",
-    "--pants-bootstrap-urls",
-    "{scie.env.PANTS_BOOTSTRAP_URLS}",
     "--ptex-path",
     "{ptex}",
     "{scie.bindings}",

package/scie-pants.toml

@@ -29,6 +29,7 @@ use crate::utils::fs::{base_name, canonicalize, copy, ensure_directory};
 
 const BINARY: &str = "scie-pants";
 
+// The version of a-scie/lift to use by default.
 const SCIENCE_TAG: &str = "v0.3.1";
 
 #[derive(Clone)]
@@ -127,7 +128,7 @@ struct Args {
         long,
         help = format!(
             "Instead of using the released {SCIENCE_TAG} science, package science from the science \
-            project repo at this directory."
+            project repo (github.com/a-scie/lift) at this directory."
         )
     )]
     science: Option<PathBuf>,

package/src/main.rs

@@ -440,7 +440,7 @@ fn test_pants_from_bad_pex_version(scie_pants_scie: &Path) {
 
     let tmpdir = create_tempdir().unwrap();
 
-    let pants_release = "2.18";
+    let pants_release = "2.19";
     let pants_toml_content = format!(
         r#"
         [GLOBAL]
@@ -459,8 +459,11 @@ fn test_pants_from_bad_pex_version(scie_pants_scie: &Path) {
     .unwrap_err();
 
     let error_text = err.to_string();
-    assert!(error_text.contains("Wasn't able to fetch the Pants PEX at"));
-    assert!(error_text.contains("Pants version format not recognized. Please add `.<patch_version>` to the end of the version. For example: `2.18` -> `2.18.0`"));
+    assert!(error_text
+        .contains("Pants version must be a full version, including patch level, got: `2.19`."));
+    assert!(error_text.contains(
+        "Please add `.<patch_version>` to the end of the version. For example: `2.18` -> `2.18.0`."
+    ));
 }
 
 fn test_use_in_repo_with_pants_script(scie_pants_scie: &Path, clone_root: &TempDir) {
@@ -1201,7 +1204,10 @@ fn test_pants_bootstrap_urls(scie_pants_scie: &Path) {
 
     assert_stderr_output(
         &mut command,
-        vec![&format!("Failed to fetch {doesnt_exist_pex_url}")],
+        vec![
+            &format!("Failed to determine release URL for Pants: {pants_release}: pants.{pants_release}-cp3"),
+            &format!(".pex: URL check failed, from PANTS_BOOTSTRAP_URLS: {doesnt_exist_pex_url}: <urlopen error [Errno 2] No such file or directory: "),
+        ],
         ExpectedResult::Failure,
     );
 

package/src/test.rs

@@ -28,7 +28,7 @@ quickly.
 One is the ability to fetch binaries check-summed in advance just in time and exactly once.
 It does this with a `"ptex"` configuration described [here](
 https://github.com/a-scie/ptex/blob/main/README.md#how-ptex-works) and employed in both the [`pbt`](
-../package/pbt.lift.json) binary and the [`scie-jump`](../package/scie-pants.lift.json) itself. This
+../package/pbt.toml) binary and the [`scie-pants`](../package/scie-pants.toml) itself. This
 allows the `scie-pants` to ship as a small binary that lazily fetches a Python distribution to run
 the tools with / install Pants with. This also gives Python tools code access to a `ptex` binary
 to use when fetching content from the internet. Although the Python code can always reach out to
@@ -39,9 +39,9 @@ The other useful feature of a scie is the ability to run pre-requisite binding c
 steps needed to support user-facing commands. These commands will only ever run once and they can
 record result information in the form of `<key>=<value>` pairs for future invocations to learn the
 one-time binding results. The `"pants"` command in the `scie-pants` binary depends on an `install`
-binding to learn the Path of the Pants venv and that in turn depends on a `configuration` binding 
-to learn the Pants version to install and the find-links repo needed to support that install. In
-turn, both of these bindings depend on Python distributions fetched by the `ptex` fetch mechanism.
+binding to learn the Path of the Pants venv and that in turn depends on a `configuration` binding to
+learn the Pants version to install. In turn, both of these bindings depend on Python distributions
+fetched by the `ptex` fetch mechanism.
 
 All such dependencies are expressed with `{scie.*}` placeholders in the values of command `"exe"`,
 `"args"` and `"env"` values in the scie lift manifest. To follow the flow of a top-level command,
@@ -67,4 +67,4 @@ python {tools.pex} <console script name> <arguments> ...
 Tools may have all of their arguments defined in the lift manifest, or they may take additional
 arguments from the user. If they take additional arguments, it's a good idea to suppress displaying
 help for the arguments defined in the lift manifest since the user can not change those and should
-probably not be concerned with their (hidden) existence.
+probably not be concerned with their (hidden) existence.

tools/README.md

@@ -63,6 +63,11 @@ def main() -> NoReturn:
     parser.add_argument(
         "--github-api-bearer-token", help="The GITHUB_TOKEN to use if running in CI context."
     )
+    parser.add_argument(
+        "--pants-bootstrap-urls",
+        type=str,
+        help="The path to the JSON file containing alternate URLs for downloaded artifacts.",
+    )
     parser.add_argument("base_dir", nargs=1, help="The base directory to create Pants venvs in.")
     options = parser.parse_args()
 
@@ -86,8 +91,8 @@ def main() -> NoReturn:
             pants_version=options.pants_version,
             find_links_dir=find_links_dir,
             github_api_bearer_token=options.github_api_bearer_token,
+            bootstrap_urls_path=options.pants_bootstrap_urls,
         )
-        version = resolve_info.stable_version
     else:
         if pants_config:
             if not prompt_for_pants_version(options.pants_config):
@@ -104,23 +109,22 @@ def main() -> NoReturn:
             pants_config=pants_config,
             find_links_dir=find_links_dir,
             github_api_bearer_token=options.github_api_bearer_token,
+            bootstrap_urls_path=options.pants_bootstrap_urls,
         )
         finalizers.append(configure_version)
-        version = resolve_info.stable_version
-
-    # N.B.: These values must match the lift TOML interpreter ids.
-    python = "cpython38" if version < Version("2.5") else "cpython39"
 
     for finalizer in finalizers:
         finalizer()
 
     with open(env_file, "a") as fp:
+        print(f"PANTS_VERSION={resolve_info.version}", file=fp)
+        print(f"PYTHON={resolve_info.python}", file=fp)
+        if resolve_info.pex_url:
+            print(f"PANTS_PEX_URL={resolve_info.pex_url}", file=fp)
         if resolve_info.find_links:
             print(f"FIND_LINKS={resolve_info.find_links}", file=fp)
         if newly_created_build_root:
             print(f"PANTS_BUILDROOT_OVERRIDE={newly_created_build_root}", file=fp)
-        print(f"PANTS_VERSION={version}", file=fp)
-        print(f"PYTHON={python}", file=fp)
 
     sys.exit(0)