diff --git a/Cargo.lock b/Cargo.lock index a6a5cdd..94fd3a8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -354,6 +354,12 @@ dependencies = [ "windows-sys 0.45.0", ] +[[package]] +name = "itoa" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" + [[package]] name = "lazy_static" version = "1.4.0" @@ -459,6 +465,7 @@ dependencies = [ "once_cell", "pretty_env_logger", "regex", + "serde_json", "sha2", "tempfile", "termcolor", @@ -591,6 +598,12 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "ryu" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" + [[package]] name = "same-file" version = "1.0.6" @@ -637,6 +650,17 @@ dependencies = [ "syn 2.0.29", ] +[[package]] +name = "serde_json" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" +dependencies = [ + "itoa", + "ryu", + "serde", +] + [[package]] name = "serde_spanned" version = "0.6.3" diff --git a/README.md b/README.md index 64808ae..3b6f27c 100644 --- a/README.md +++ b/README.md @@ -60,48 +60,50 @@ provides the following: `scie-pants` executable to `pants_from_sources` and execute that. In this case `PANTS_SOURCE` will default to `../pants` just as was the case in the bespoke `./pants_from_sources` scripts. -+ Partial support for firewalls: - - Currently, you can re-direct the URLs used to fetch: - - + [Python Build Standalone](https://python-build-standalone.readthedocs.io/en/latest/) CPython - distributions used to bootstrap Pants. - + Pants PEX release assets which contain Pants as a single-file application. - - This is done by exporting a `PANTS_BOOTSTRAP_URLS` environment variable - specifying the path to a JSON file containing a mapping of file names to URLS to fetch them from - under a top-level `"ptex"` key. For example: - ```json - { - "ptex": { - "cpython-3.8.16+20230507-x86_64-unknown-linux-gnu-install_only.tar.gz": "https://example.com/cpython-3.8.16%2B20230507-x86_64-unknown-linux-gnu-install_only.tar.gz", - "cpython-3.8.16+20230507-aarch64-apple-darwin-install_only.tar.gz": "https://example.com/cpython-3.8.16%2B20230507-aarch64-apple-darwin-install_only.tar.gz", - "cpython-3.9.16+20230507-x86_64-unknown-linux-gnu-install_only.tar.gz": "https://example.com/cpython-3.9.16%2B20230507-x86_64-unknown-linux-gnu-install_only.tar.gz", - "cpython-3.9.16+20230507-aarch64-apple-darwin-install_only.tar.gz": "https://example.com/cpython-3.9.16%2B20230507-aarch64-apple-darwin-install_only.tar.gz", - "pants.2.18.0-cp9-linux-x86_64.pex": "https://example.com/pants.2.18.0-cp9-linux-x86_64.pex", - ... - } ++ Partial support for firewalls + +### Firewall support + +Currently, you can re-direct the URLs used to fetch: + + + [Python Build Standalone](https://python-build-standalone.readthedocs.io/en/latest/) CPython + distributions used to bootstrap Pants. + + Pants PEX release assets which contain Pants as a single-file application. + +This is done by exporting a `PANTS_BOOTSTRAP_URLS` environment variable +specifying the path to a JSON file containing a mapping of file names to URLS to fetch them from +under a top-level `"ptex"` key. For example: +```json +{ + "ptex": { + "cpython-3.8.16+20230507-x86_64-unknown-linux-gnu-install_only.tar.gz": "https://example.com/cpython-3.8.16%2B20230507-x86_64-unknown-linux-gnu-install_only.tar.gz", + "cpython-3.8.16+20230507-aarch64-apple-darwin-install_only.tar.gz": "https://example.com/cpython-3.8.16%2B20230507-aarch64-apple-darwin-install_only.tar.gz", + "cpython-3.9.16+20230507-x86_64-unknown-linux-gnu-install_only.tar.gz": "https://example.com/cpython-3.9.16%2B20230507-x86_64-unknown-linux-gnu-install_only.tar.gz", + "cpython-3.9.16+20230507-aarch64-apple-darwin-install_only.tar.gz": "https://example.com/cpython-3.9.16%2B20230507-aarch64-apple-darwin-install_only.tar.gz", + "pants.2.18.0-cp39-linux_x86_64.pex": "https://example.com/pants.2.18.0-cp39-linux_x86_64.pex", + ... } - ``` - - For keys that are "embedded" into `scie-pants` itself (such as Python Build Standalone), you can run: - ``` - $ SCIE=inspect scie-pants | jq .ptex - ``` - You'll need to run this once for each platform you use `scie-pants` on to gather all mappings - you'll need; e.g.: once for Linux x86_64 and once for Mac ARM. - - The embedded artifact references also contain expected hashes of the downloaded content. Your - re-directed URLs must provide the same content as the canonical URLs; if the hashes of downloaded - files do not match those recorded in `scie-pants`, install will fail fast and let you know about - the hash mismatch. - - For other keys that aren't embedded, and are generated on-the-fly (such as the Pants PEX), there - is no single source of truth that can be easily scraped out. For the Pants PEX, the key is the versioned - PEX name (E.g. `pants.---.pex`). These can be found on the relevant - GitHub Release page's Assets (e.g. https://github.com/pantsbuild/pants/releases/tag/release_2.18.0a0). - (Note that for 2.18.x, PEX exist versioned and unversioned. `scie-pants` only uses the versioned - name as the key). +} +``` + +For keys that are "embedded" into `scie-pants` itself (such as Python Build Standalone), you can run: +``` +$ SCIE=inspect scie-pants | jq .ptex +``` +You'll need to run this once for each platform you use `scie-pants` on to gather all mappings +you'll need; e.g.: once for Linux x86_64 and once for Mac ARM. + +The embedded artifact references also contain expected hashes of the downloaded content. Your +re-directed URLs must provide the same content as the canonical URLs; if the hashes of downloaded +files do not match those recorded in `scie-pants`, install will fail fast and let you know about +the hash mismatch. + +For other keys that aren't embedded, and are generated on-the-fly (such as the Pants PEX), there +is no single source of truth that can be easily scraped out. For the Pants PEX, the key is the versioned +PEX name (E.g. `pants.--_.pex`). These can be found on the relevant +GitHub Release page's Assets (e.g. https://github.com/pantsbuild/pants/releases/tag/release_2.18.0a0). +(Note that for 2.18.x, PEX exist versioned and unversioned. `scie-pants` only uses the versioned +name as the key). ## Caveats diff --git a/package/Cargo.toml b/package/Cargo.toml index 436a610..3a7c1de 100644 --- a/package/Cargo.toml +++ b/package/Cargo.toml @@ -18,6 +18,7 @@ log = { workspace = true } once_cell = "1.18" pretty_env_logger = "0.5" regex = "1.9" +serde_json = "1.0.107" sha2 = "0.10" tempfile = { workspace = true } termcolor = "1.3" diff --git a/package/src/test.rs b/package/src/test.rs index 01a4525..224fb8d 100644 --- a/package/src/test.rs +++ b/package/src/test.rs @@ -37,8 +37,32 @@ fn decode_output(output: Vec) -> Result { String::from_utf8(output).context("Failed to decode Pants output.") } -fn assert_stderr_output(command: &mut Command, expected_messages: Vec<&str>) -> Output { - let output = execute(command.stderr(Stdio::piped())).unwrap(); +enum ExpectedResult { + Success, + Failure, +} + +fn assert_stderr_output( + command: &mut Command, + expected_messages: Vec<&str>, + expected_result: ExpectedResult, +) -> (Output, String) { + command.stderr(Stdio::piped()); + + let output = match expected_result { + ExpectedResult::Success => execute(command).unwrap(), + ExpectedResult::Failure => { + let output = command.spawn().unwrap().wait_with_output().unwrap(); + assert!( + !output.status.success(), + "Command {:?} unexpectedly succeeded, STDERR: {}", + command, + decode_output(output.stderr).unwrap() + ); + output + } + }; + let stderr = decode_output(output.stderr.clone()).unwrap(); for expected_message in expected_messages { assert!( @@ -46,7 +70,7 @@ fn assert_stderr_output(command: &mut Command, expected_messages: Vec<&str>) -> "STDERR did not contain '{expected_message}':\n{stderr}" ); } - output + (output, stderr) } pub(crate) fn run_integration_tests( @@ -139,6 +163,7 @@ pub(crate) fn run_integration_tests( test_non_utf8_env_vars_issue_198(scie_pants_scie); test_bad_boot_error_text(scie_pants_scie); + test_pants_bootstrap_urls(scie_pants_scie); } // Max Python supported is 3.8 and only Linux and macOS x86_64 wheels were released. @@ -628,6 +653,7 @@ index b70ae75..271706a 100644 "The PANTS_SOURCE mode is working.", "Pants from sources argv: --no-verify-config -V.", ], + ExpectedResult::Success, ); } @@ -658,6 +684,7 @@ fn test_pants_from_sources_mode( "The pants_from_sources mode is working.", "Pants from sources argv: --no-verify-config -V.", ], + ExpectedResult::Success, ); } @@ -672,13 +699,14 @@ fn test_delegate_pants_in_pants_repo(scie_pants_scie: &Path, pants_2_14_1_clone_ "The delegate_bootstrap mode is working.", "Pants from sources argv: -V.", ], + ExpectedResult::Success, ); } fn test_use_pants_release_in_pants_repo(scie_pants_scie: &Path, pants_2_14_1_clone_dir: &PathBuf) { let pants_release = "2.16.0rc2"; integration_test!("Verify usage of Pants {pants_release} on the pants repo."); - let output = assert_stderr_output( + let (output, stderr) = assert_stderr_output( Command::new(scie_pants_scie) .arg("help") .env("PANTS_VERSION", pants_release) @@ -692,6 +720,7 @@ fn test_use_pants_release_in_pants_repo(scie_pants_scie: &Path, pants_2_14_1_clo .current_dir(pants_2_14_1_clone_dir) .stdout(Stdio::piped()), vec![], + ExpectedResult::Success, ); let expected_message = pants_release; let stdout = decode_output(output.stdout).unwrap(); @@ -700,7 +729,6 @@ fn test_use_pants_release_in_pants_repo(scie_pants_scie: &Path, pants_2_14_1_clo "STDOUT did not contain '{expected_message}':\n{stdout}" ); let unexpected_message = "Pants from sources argv"; - let stderr = decode_output(output.stderr).unwrap(); assert!( !stderr.contains(unexpected_message), "STDERR unexpectedly contained '{unexpected_message}':\n{stderr}" @@ -1028,27 +1056,17 @@ fn test_bad_boot_error_text(scie_pants_scie: &Path) { integration_test!( "Verifying the output of scie-pants is user-friendly if they provide an unexpected SCIE_BOOT argument", ); - // (Avoids `execute` because this is expected to fail.) - let child = Command::new(scie_pants_scie) - .env("SCIE_BOOT", "does-not-exist") - .stderr(Stdio::piped()) - .spawn() - .unwrap(); - let output = child.wait_with_output().unwrap(); - let stderr = decode_output(output.stderr).unwrap(); - - for expected in [ - "`SCIE_BOOT=does-not-exist` was found in the environment", - // the various boot commands we want users to know about - "\n ", - "\nbootstrap-tools ", - "\nupdate ", - ] { - assert!( - stderr.contains(expected), - "STDERR does not contain '{expected:?}':\n{stderr}" - ); - } + let (_, stderr) = assert_stderr_output( + Command::new(scie_pants_scie).env("SCIE_BOOT", "does-not-exist"), + vec![ + "`SCIE_BOOT=does-not-exist` was found in the environment", + // the various boot commands we want users to know about + "\n ", + "\nbootstrap-tools ", + "\nupdate ", + ], + ExpectedResult::Failure, + ); // Check that boot commands that users shouldn't see (used internally, only) aren't included. for bad_boot in ["pants", "pants-debug"] { @@ -1059,3 +1077,119 @@ fn test_bad_boot_error_text(scie_pants_scie: &Path) { ); } } + +fn test_pants_bootstrap_urls(scie_pants_scie: &Path) { + integration_test!( + "Verifying PANTS_BOOTSTRAP_URLS is used for both CPython interpreter and Pants PEX ({issue})", + issue = issue_link(243) + ); + + // This test runs in 4 parts: + // + // 0. Setup tempdirs, common values etc. + // 1. Verify interpreter download uses URL (by checking errors with a non-existent URL) + // 2. The same, but for the Pants PEX + // 3. Verify that specifying valid URLs works too (no good if we're just succesfully failing) + + // Part 0: Setup + let tmpdir = create_tempdir().unwrap(); + + // A fresh directory to ensure the downloads happen fresh. + let scie_base = tmpdir.path().join("scie-base"); + + // The file that we'll plop our URL overrides into... + let urls_json = tmpdir.path().join("urls.json"); + // ... plus helpers to write to it, we start with the `ptex` key/value of this scie-pants's + // `SCIE=inspect` output (which will be the Python interpreters and their default URLs), but + // allow the tests to update it. + let output = execute( + Command::new(scie_pants_scie) + .env("SCIE", "inspect") + .stdout(Stdio::piped()), + ) + .unwrap(); + let mut ptex_json: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); + ptex_json + .as_object_mut() + .unwrap() + .retain(|key, _| key == "ptex"); + + let write_urls_json = + |update_ptex_map: &dyn Fn(&mut serde_json::Map)| { + let mut json = ptex_json.clone(); + // Transform the "ptex": {...} map as appropriate. + update_ptex_map(json["ptex"].as_object_mut().unwrap()); + write_file(&urls_json, false, serde_json::to_vec(&json).unwrap()).unwrap(); + }; + + // Reference data for the Pants we'll try to install (NB. we have to force new-enough version of + // Pants to install via PEXes, older versions go via PyPI which isn't managed by + // PANTS_BOOTSTRAP_URLS) + let pants_release = "2.18.0rc1"; + let platforms = [ + "darwin_arm64", + "darwin_x86_64", + "linux_aarch64", + "linux_x86_64", + ]; + let pexes = platforms + .iter() + .map(|platform| format!("pants.{pants_release}-cp39-{platform}.pex")) + .collect::>(); + + // we run the exact same command each time + let mut command = Command::new(scie_pants_scie); + command + .arg("-V") + .env("PANTS_BOOTSTRAP_URLS", &urls_json) + .env("SCIE_BASE", &scie_base) + .env("PANTS_VERSION", pants_release); + + // Part 1: Validate that we attempt to download the CPython interpreter from (invalid) override + // URLs + + // Set every ptex file value to the substitute URL, that doesn't exist + let doesnt_exist_interpreter = tmpdir.path().join("doesnt-exist-interpreter"); + let doesnt_exist_interpreter_url = format!("file://{}", doesnt_exist_interpreter.display()); + write_urls_json(&|ptex_map| { + for value in ptex_map.values_mut() { + *value = doesnt_exist_interpreter_url.clone().into(); + } + }); + + assert_stderr_output( + &mut command, + vec![&format!("Failed to fetch {doesnt_exist_interpreter_url}")], + ExpectedResult::Failure, + ); + + // Part 2: Validate that we attempt to download Pants PEXes from (invalid) override URLs + + // Leave the interpreters and add new URLs for the various PEXes that this test might need (all + // the different platforms); as above, the URL doesn't exist + let doesnt_exist_pex = tmpdir.path().join("doesnt-exist-pex"); + let doesnt_exist_pex_url = format!("file://{}", doesnt_exist_pex.display()); + write_urls_json(&|ptex_map| { + for pex in &pexes { + ptex_map.insert(pex.clone(), doesnt_exist_pex_url.clone().into()); + } + }); + + assert_stderr_output( + &mut command, + vec![&format!("Failed to fetch {doesnt_exist_pex_url}")], + ExpectedResult::Failure, + ); + + // Part 3: Validate that we can bootstrap pants fully from these override URLs (by manually + // re-specifying the defaults) + write_urls_json(&|ptex_map| { + for pex in &pexes { + ptex_map.insert(pex.clone(), format!("https://github.com/pantsbuild/pants/releases/download/release_{pants_release}/{pex}").into()); + } + }); + + let output = execute(&mut command.stdout(Stdio::piped())).unwrap(); + let stdout = decode_output(output.stdout).unwrap(); + assert!(stdout.contains(pants_release)); +}