chore: add timeout to spec tests

tests/specs/mod.rs

@@ -129,6 +129,8 @@ struct MultiStepMetaData {
   #[serde(default)]
   pub envs: HashMap<String, String>,
   #[serde(default)]
+  pub timeout: Option<usize>,
+  #[serde(default)]
   pub repeat: Option<usize>,
   #[serde(default)]
   pub steps: Vec<StepMetaData>,
@@ -146,6 +148,8 @@ struct SingleTestMetaData {
   #[serde(default)]
   pub symlinked_temp_dir: bool,
   #[serde(default)]
+  pub timeout: Option<usize>,
+  #[serde(default)]
   pub repeat: Option<usize>,
   #[serde(flatten)]
   pub step: StepMetaData,
@@ -161,6 +165,7 @@ impl SingleTestMetaData {
       temp_dir: self.temp_dir,
       symlinked_temp_dir: self.symlinked_temp_dir,
       repeat: self.repeat,
+      timeout: self.timeout,
       envs: Default::default(),
       steps: vec![self.step],
       ignore: self.ignore,
@@ -285,8 +290,9 @@ fn run_test_inner(
   diagnostic_logger: Rc<RefCell<Vec<u8>>>,
 ) {
   let context = test_context_from_metadata(metadata, cwd, diagnostic_logger);
+  let start_time = std::time::Instant::now();
   for step in metadata.steps.iter().filter(|s| should_run_step(s)) {
-    let run_func = || run_step(step, metadata, cwd, &context);
+    let run_func = || run_step(step, metadata, cwd, &context, start_time);
     if step.flaky {
       run_flaky(run_func);
     } else {
@@ -395,6 +401,7 @@ fn run_step(
   metadata: &MultiStepMetaData,
   cwd: &PathRef,
   context: &test_util::TestContext,
+  start_time: std::time::Instant,
 ) {
   let command = context
     .new_command()
@@ -421,6 +428,16 @@ fn run_step(
     Some(input) => command.stdin_text(input),
     None => command,
   };
+  let command = match metadata.timeout {
+    Some(timeout_seconds) => {
+      let timeout = std::time::Duration::from_secs(timeout_seconds as u64);
+      let remaining_timeout = timeout
+        .checked_sub(start_time.elapsed())
+        .unwrap_or_else(|| panic!("Timed out after: {}s", timeout_seconds));
+      command.timeout(remaining_timeout)
+    }
+    None => command,
+  };
   let output = command.run();
   if step.output.ends_with(".out") {
     let test_output_path = cwd.join(&step.output);

tests/specs/schema.json

@@ -75,17 +75,21 @@
               "type": "string"
             }
           },
+          "ignore": {
+            "type": "boolean"
+          },
           "repeat": {
             "type": "number"
           },
+          "timeout": {
+            "description": "Number of seconds to wait before timing out the test.",
+            "type": "number"
+          },
           "steps": {
             "type": "array",
             "items": {
               "$ref": "#/definitions/single_test"
             }
-          },
-          "ignore": {
-            "type": "boolean"
           }
         }
       }, {
@@ -105,6 +109,10 @@
             },
             "repeat": {
               "type": "number"
+            },
+            "timeout": {
+              "description": "Number of seconds to wait before timing out the test.",
+              "type": "number"
             }
           }
         }, {
@@ -127,17 +135,21 @@
             "type": "string"
           }
         },
+        "ignore": {
+          "type": "boolean"
+        },
         "repeat": {
           "type": "number"
         },
+        "timeout": {
+          "description": "Number of seconds to wait before timing out the test.",
+          "type": "number"
+        },
         "tests": {
           "type": "object",
           "additionalProperties": {
             "$ref": "#/definitions/single_or_multi_step_test"
           }
-        },
-        "ignore": {
-          "type": "boolean"
         }
       }
     }

tests/util/server/src/builders.rs

@@ -30,6 +30,7 @@ use crate::jsr_registry_unset_url;
 use crate::lsp::LspClientBuilder;
 use crate::nodejs_org_mirror_unset_url;
 use crate::npm_registry_unset_url;
+use crate::process::wait_on_child_with_timeout;
 use crate::pty::Pty;
 use crate::strip_ansi_codes;
 use crate::testdata_path;
@@ -412,6 +413,7 @@ pub struct TestCommandBuilder {
   args_vec: Vec<String>,
   split_output: bool,
   show_output: bool,
+  timeout: Option<std::time::Duration>,
 }
 
 impl TestCommandBuilder {
@@ -432,6 +434,7 @@ impl TestCommandBuilder {
       args_text: "".to_string(),
       args_vec: Default::default(),
       show_output: false,
+      timeout: None,
     }
   }
 
@@ -542,6 +545,11 @@ impl TestCommandBuilder {
     self
   }
 
+  pub fn timeout(mut self, timeout: std::time::Duration) -> Self {
+    self.timeout = Some(timeout);
+    self
+  }
+
   pub fn stdin_piped(self) -> Self {
     self.stdin(std::process::Stdio::piped())
   }
@@ -599,6 +607,8 @@ impl TestCommandBuilder {
       }
     }
 
+    assert!(self.timeout.is_none(), "not implemented");
+
     let command_path = self.build_command_path();
 
     self.diagnostic_logger.writeln(format!(
@@ -614,15 +624,18 @@ impl TestCommandBuilder {
 
   pub fn output(&self) -> Result<std::process::Output, std::io::Error> {
     assert!(self.stdin_text.is_none(), "use spawn instead");
+    assert!(self.timeout.is_none(), "not implemented");
     self.build_command().output()
   }
 
   pub fn status(&self) -> Result<std::process::ExitStatus, std::io::Error> {
     assert!(self.stdin_text.is_none(), "use spawn instead");
+    assert!(self.timeout.is_none(), "not implemented");
     self.build_command().status()
   }
 
   pub fn spawn(&self) -> Result<DenoChild, std::io::Error> {
+    assert!(self.timeout.is_none(), "not implemented");
     let child = self.build_command().spawn()?;
     let mut child = DenoChild {
       _deno_dir: self.deno_dir.clone(),
@@ -681,7 +694,7 @@ impl TestCommandBuilder {
       .get_args()
       .map(ToOwned::to_owned)
       .collect::<Vec<_>>();
-    let (combined_reader, std_out_err_handle) = if self.split_output {
+    let (combined_handle, std_out_err_handle) = if self.split_output {
       let (stdout_reader, stdout_writer) = pipe().unwrap();
       let (stderr_reader, stderr_writer) = pipe().unwrap();
       command.stdout(stdout_writer);
@@ -699,10 +712,14 @@ impl TestCommandBuilder {
         )),
       )
     } else {
+      let show_output = self.show_output;
       let (combined_reader, combined_writer) = pipe().unwrap();
       command.stdout(combined_writer.try_clone().unwrap());
       command.stderr(combined_writer);
-      (Some(combined_reader), None)
+      let combined_handle = std::thread::spawn(move || {
+        read_pipe_to_string(combined_reader, show_output)
+      });
+      (Some(combined_handle), None)
     };
 
     let mut process = command.spawn().expect("Failed spawning command");
@@ -718,17 +735,18 @@ impl TestCommandBuilder {
     // and dropping it closes them.
     drop(command);
 
-    let combined = combined_reader.map(|pipe| {
-      sanitize_output(read_pipe_to_string(pipe, self.show_output), &args)
-    });
-
-    let status = process.wait().unwrap();
+    let status = match self.timeout {
+      Some(timeout) => wait_on_child_with_timeout(process, timeout).unwrap(),
+      None => process.wait().unwrap(),
+    };
     let std_out_err = std_out_err_handle.map(|(stdout, stderr)| {
       (
         sanitize_output(stdout.join().unwrap(), &args),
         sanitize_output(stderr.join().unwrap(), &args),
       )
     });
+    let combined = combined_handle
+      .map(|handle| sanitize_output(handle.join().unwrap(), &args));
     let exit_code = status.code();
     #[cfg(unix)]
     let signal = {

tests/util/server/src/lib.rs

@@ -31,6 +31,7 @@ mod https;
 pub mod lsp;
 mod macros;
 mod npm;
+mod process;
 pub mod pty;
 pub mod servers;
 pub mod spawn;

tests/util/server/src/process.rs

@@ -0,0 +1,81 @@
+// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
+
+use std::process::Child;
+use std::process::ExitStatus;
+
+use anyhow::bail;
+
+pub fn wait_on_child_with_timeout(
+  mut child: Child,
+  timeout: std::time::Duration,
+) -> Result<ExitStatus, anyhow::Error> {
+  let (tx, rx) = std::sync::mpsc::channel();
+  let pid = child.id();
+
+  std::thread::spawn(move || {
+    let status = child.wait().unwrap();
+    _ = tx.send(status);
+  });
+
+  match rx.recv_timeout(timeout) {
+    Ok(status) => Ok(status),
+    Err(_) => {
+      kill(pid as i32)?;
+      bail!(
+        "Child process timed out after {}s",
+        timeout.as_secs_f32().ceil() as u64
+      );
+    }
+  }
+}
+
+#[cfg(unix)]
+pub fn kill(pid: i32) -> Result<(), anyhow::Error> {
+  use nix::sys::signal::kill as unix_kill;
+  use nix::sys::signal::Signal;
+  use nix::unistd::Pid;
+  let sig = Signal::SIGTERM;
+  Ok(unix_kill(Pid::from_raw(pid), Some(sig))?)
+}
+
+#[cfg(not(unix))]
+pub fn kill(pid: i32) -> Result<(), anyhow::Error> {
+  use std::io::Error;
+  use std::io::ErrorKind::NotFound;
+  use winapi::shared::minwindef::DWORD;
+  use winapi::shared::minwindef::FALSE;
+  use winapi::shared::minwindef::TRUE;
+  use winapi::shared::winerror::ERROR_INVALID_PARAMETER;
+  use winapi::um::errhandlingapi::GetLastError;
+  use winapi::um::handleapi::CloseHandle;
+  use winapi::um::processthreadsapi::OpenProcess;
+  use winapi::um::processthreadsapi::TerminateProcess;
+  use winapi::um::winnt::PROCESS_TERMINATE;
+
+  if pid <= 0 {
+    bail!("Invalid pid: {}", pid);
+  }
+  let handle =
+      // SAFETY: winapi call
+      unsafe { OpenProcess(PROCESS_TERMINATE, FALSE, pid as DWORD) };
+
+  if handle.is_null() {
+    // SAFETY: winapi call
+    let err = match unsafe { GetLastError() } {
+      ERROR_INVALID_PARAMETER => Error::from(NotFound), // Invalid `pid`.
+      errno => Error::from_raw_os_error(errno as i32),
+    };
+    Err(err.into())
+  } else {
+    // SAFETY: winapi calls
+    unsafe {
+      let is_terminated = TerminateProcess(handle, 1);
+      CloseHandle(handle);
+      match is_terminated {
+        FALSE => Err(Error::last_os_error().into()),
+        TRUE => Ok(()),
+        _ => unreachable!(),
+      }
+    }
+  }
+}