diff --git a/package/src/test.rs b/package/src/test.rs
index 4a8d95c..b369f2d 100644
--- a/package/src/test.rs
+++ b/package/src/test.rs
@@ -13,7 +13,7 @@ use tempfile::TempDir;
 use termcolor::{Color, WriteColor};
 
 use crate::utils::build::fingerprint;
-use crate::utils::exe::{execute, execute_with_input, Platform, CURRENT_PLATFORM};
+use crate::utils::exe::{execute, execute_with_input, execute_no_error, Platform, CURRENT_PLATFORM};
 use crate::utils::fs::{
     copy, create_tempdir, ensure_directory, remove_dir, rename, softlink, touch, write_file,
 };
@@ -38,7 +38,7 @@ fn decode_output(output: Vec<u8>) -> Result<String> {
 }
 
 fn assert_stderr_output(command: &mut Command, expected_messages: Vec<&str>) -> Output {
-    let output = execute(command.stderr(Stdio::piped())).unwrap();
+    let output = execute_no_error(command.stderr(Stdio::piped()));
     let stderr = decode_output(output.stderr.clone()).unwrap();
     for expected_message in expected_messages {
         assert!(
@@ -103,6 +103,7 @@ pub(crate) fn run_integration_tests(
         test_initialize_new_pants_project(scie_pants_scie);
         test_set_pants_version(scie_pants_scie);
         test_ignore_empty_pants_version_pants_sha(scie_pants_scie);
+        test_bad_pants_version(scie_pants_scie);
 
         let clone_root = create_tempdir()?;
         test_use_in_repo_with_pants_script(scie_pants_scie, &clone_root);
@@ -392,6 +393,28 @@ fn test_ignore_empty_pants_version_pants_sha(scie_pants_scie: &Path) {
     );
 }
 
+fn test_bad_pants_version(scie_pants_scie: &Path) {
+    integration_test!("Verifying Pants gives good error messages for nonexistent version numbers");
+    let non_existent_version = "1.2.3.4.5";
+    assert_stderr_output(
+        Command::new(scie_pants_scie)
+            .arg("-V")
+            .env("PANTS_VERSION", non_existent_version),
+        vec![
+            "Could not find Pants tag 1.2.3.4.5 in https://github.com/pantsbuild/pants/releases"
+        ],
+    );
+    let prefix_version = "1";
+    assert_stderr_output(
+        Command::new(scie_pants_scie)
+            .arg("-V")
+            .env("PANTS_VERSION", prefix_version),
+        vec![
+            "Could not find Pants tag 1 in https://github.com/pantsbuild/pants/releases"
+        ],
+    );
+}
+
 fn test_use_in_repo_with_pants_script(scie_pants_scie: &Path, clone_root: &TempDir) {
     integration_test!("Verify scie-pants can be used as `pants` in a repo with the `pants` script");
     // This verifies a fix for https://github.com/pantsbuild/scie-pants/issues/28.
diff --git a/package/src/utils/exe.rs b/package/src/utils/exe.rs
index 8e7b916..2737e31 100644
--- a/package/src/utils/exe.rs
+++ b/package/src/utils/exe.rs
@@ -81,14 +81,22 @@ pub(crate) fn prepare_exe(path: &Path) -> Result<()> {
 }
 
 pub(crate) fn execute_with_input(command: &mut Command, stdin_data: &[u8]) -> Result<Output> {
-    _execute_with_input(command, Some(stdin_data))
+    _execute_with_input(command, Some(stdin_data), true)
 }
 
 pub(crate) fn execute(command: &mut Command) -> Result<Output> {
-    _execute_with_input(command, None)
+    _execute_with_input(command, None, true)
 }
 
-fn _execute_with_input(command: &mut Command, stdin_data: Option<&[u8]>) -> Result<Output> {
+pub(crate) fn execute_no_error(command: &mut Command) -> Output {
+    _execute_with_input(command, None, false).unwrap()
+}
+
+fn _execute_with_input(
+    command: &mut Command,
+    stdin_data: Option<&[u8]>,
+    err_on_command_fail: bool,
+) -> Result<Output> {
     info!("Executing {command:#?}");
     if stdin_data.is_some() {
         command.stdin(std::process::Stdio::piped());
@@ -107,7 +115,7 @@ fn _execute_with_input(command: &mut Command, stdin_data: Option<&[u8]>) -> Resu
     let output = child
         .wait_with_output()
         .with_context(|| format!("Failed to gather exit status of command: {command:?}"))?;
-    if !output.status.success() {
+    if err_on_command_fail && !output.status.success() {
         let mut message_lines = vec![format!(
             "Command {command:?} failed with exit code: {code:?}",
             code = output.status.code()