Adding max-run-time argument (#306)

* fixing tests and addind arg.

* Adding tests.

* Running formatter.

* Incrementing version.

Author: EmbeddedDevops1

cover_agent/CoverAgent.py

@@ -13,7 +13,6 @@
 from cover_agent.AICaller import AICaller
 from cover_agent.AgentCompletionABC import AgentCompletionABC
 from cover_agent.DefaultAgentCompletion import DefaultAgentCompletion
-import cover_agent.utils
 
 
 class CoverAgent:
@@ -106,6 +105,7 @@ def __init__(self, args, agent_completion: AgentCompletionABC = None):
             comparison_branch=args.branch,
             num_attempts=args.run_tests_multiple_times,
             agent_completion=self.agent_completion,
+            max_run_time=args.max_run_time,
         )
 
     def _validate_paths(self):

cover_agent/Runner.py

@@ -1,41 +1,35 @@
 import subprocess
 import time
 
-from cover_agent.settings.config_loader import get_settings
-
 
 class Runner:
     @staticmethod
-    def run_command(command, cwd=None):
+    def run_command(command: str, max_run_time: int, cwd: str = None):
         """
         Executes a shell command in a specified working directory and returns its output, error, and exit code.
 
         Parameters:
             command (str): The shell command to execute.
+            max_run_time (int): Maximum allowed runtime in seconds before timeout.
             cwd (str, optional): The working directory in which to execute the command. Defaults to None.
 
         Returns:
-            tuple: A tuple containing the standard output ('stdout'), standard error ('stderr'), exit code ('exit_code'), and the time of the executed command ('command_start_time').
+            tuple: A tuple containing the standard output ('stdout'), standard error ('stderr'), exit code ('exit_code'),
+                   and the time of the executed command ('command_start_time').
         """
-        # Get the current time before running the test command, in milliseconds
-        command_start_time = int(round(time.time() * 1000))
+        command_start_time = int(
+            time.time() * 1000
+        )  # Get the current time in milliseconds
 
-        max_allowed_runtime_seconds = get_settings().get(
-            "tests.max_allowed_runtime_seconds", 3600
-        )
-        # Ensure the command is executed with shell=True for string commands
         try:
             result = subprocess.run(
                 command,
                 shell=True,
                 cwd=cwd,
                 text=True,
                 capture_output=True,
-                timeout=max_allowed_runtime_seconds,
+                timeout=max_run_time,
             )
-
-            # Return a dictionary with the desired information
             return result.stdout, result.stderr, result.returncode, command_start_time
         except subprocess.TimeoutExpired:
-            # Handle the timeout case
             return "", "Command timed out", -1, command_start_time

cover_agent/UnitTestValidator.py

@@ -24,6 +24,7 @@ def __init__(
         code_coverage_report_path: str,
         test_command: str,
         llm_model: str,
+        max_run_time: int,
         agent_completion: AgentCompletionABC,
         test_command_dir: str = os.getcwd(),
         included_files: list = None,
@@ -45,6 +46,7 @@ def __init__(
             code_coverage_report_path (str): The path to the code coverage report file.
             test_command (str): The command to run tests.
             llm_model (str): The language model to be used for test generation.
+            max_run_time (int): The maximum time in seconds to run the test command.
             agent_completion (AgentCompletionABC): The agent completion object to use for test generation.
             api_base (str, optional): The base API url to use in case model is set to Ollama or Hugging Face. Defaults to an empty string.
             test_command_dir (str, optional): The directory where the test command should be executed. Defaults to the current working directory.
@@ -81,6 +83,7 @@ def __init__(
         self.comparison_branch = comparison_branch
         self.num_attempts = num_attempts
         self.agent_completion = agent_completion
+        self.max_run_time = max_run_time
 
         # Get the logger instance from CustomLogger
         self.logger = CustomLogger.get_logger(__name__)
@@ -292,7 +295,9 @@ def run_coverage(self):
             f'Running build/test command to generate coverage report: "{self.test_command}"'
         )
         stdout, stderr, exit_code, time_of_test_command = Runner.run_command(
-            command=self.test_command, cwd=self.test_command_dir
+            command=self.test_command,
+            max_run_time=self.max_run_time,
+            cwd=self.test_command_dir,
         )
         assert (
             exit_code == 0

cover_agent/main.py

@@ -71,6 +71,12 @@ def parse_args():
         default=10,
         help="The maximum number of iterations. Default: %(default)s.",
     )
+    parser.add_argument(
+        "--max-run-time",
+        type=int,
+        default=30,
+        help="Maximum time (in seconds) allowed for test execution. Overrides the value in configuration.toml if provided. Defaults to 30 seconds.",
+    )
     parser.add_argument(
         "--additional-instructions",
         default="",

cover_agent/settings/configuration.toml

@@ -3,4 +3,4 @@ limit_tokens=true
 max_tokens=20000
 
 [tests]
-max_allowed_runtime_seconds=3600
+max_allowed_runtime_seconds=30

cover_agent/version.txt

@@ -1 +1 @@
-0.3.4
+0.3.5

tests/test_CoverAgent.py

@@ -55,6 +55,7 @@ def test_agent_source_file_not_found(self, mock_isfile, mock_unit_cover_agent):
             report_filepath="test_results.html",
             desired_coverage=90,
             max_iterations=10,
+            max_run_time=30,
         )
         parse_args = lambda: args
         mock_isfile.return_value = False
@@ -88,6 +89,7 @@ def test_agent_test_file_not_found(
             desired_coverage=90,
             max_iterations=10,
             prompt_only=False,
+            max_run_time=30,
         )
         parse_args = lambda: args
         mock_isfile.side_effect = [True, False]
@@ -128,6 +130,7 @@ def test_duplicate_test_file_without_output_path(self, mock_isfile):
                     diff_coverage=False,
                     branch="main",
                     run_tests_multiple_times=1,
+                    max_run_time=30,
                 )
 
                 with pytest.raises(AssertionError) as exc_info:
@@ -183,6 +186,7 @@ def test_run_max_iterations_strict_coverage(
                 strict_coverage=True,
                 diff_coverage=False,
                 branch="main",
+                max_run_time=30,
             )
             # Mock the methods used in run
             validator = mock_unit_test_validator.return_value
@@ -215,6 +219,7 @@ def test_project_root_not_found(self, mock_isdir, mock_isfile):
             report_filepath="test_results.html",
             desired_coverage=90,
             max_iterations=10,
+            max_run_time=30,
         )
 
         with pytest.raises(FileNotFoundError) as exc_info:
@@ -259,6 +264,7 @@ def test_run_diff_coverage(
                 strict_coverage=False,
                 diff_coverage=True,
                 branch="main",
+                max_run_time=30,
             )
             mock_test_validator.return_value.current_coverage = 0.5
             mock_test_validator.return_value.desired_coverage = 90

tests/test_Runner.py

@@ -1,21 +1,23 @@
 import pytest
 from unittest.mock import patch
-from cover_agent.Runner import Runner  # Adjust the import path as necessary
+from cover_agent.Runner import Runner
 
 
 class TestRunner:
     def test_run_command_success(self):
         """Test the run_command method with a command that succeeds."""
         command = 'echo "Hello, World!"'
-        stdout, stderr, exit_code, _ = Runner.run_command(command)
+        stdout, stderr, exit_code, _ = Runner.run_command(command, max_run_time=10)
         assert stdout.strip() == "Hello, World!"
         assert stderr == ""
         assert exit_code == 0
 
     def test_run_command_with_cwd(self):
         """Test the run_command method with a specified working directory."""
         command = 'echo "Working Directory"'
-        stdout, stderr, exit_code, _ = Runner.run_command(command, cwd="/tmp")
+        stdout, stderr, exit_code, _ = Runner.run_command(
+            command, cwd="/tmp", max_run_time=10
+        )
         assert stdout.strip() == "Working Directory"
         assert stderr == ""
         assert exit_code == 0
@@ -24,10 +26,18 @@ def test_run_command_failure(self):
         """Test the run_command method with a command that fails."""
         # Use a command that is guaranteed to fail
         command = "command_that_does_not_exist"
-        stdout, stderr, exit_code, _ = Runner.run_command(command)
+        stdout, stderr, exit_code, _ = Runner.run_command(command, max_run_time=10)
         assert stdout == ""
         assert (
             "command_that_does_not_exist: not found" in stderr
             or "command_that_does_not_exist: command not found" in stderr
         )
         assert exit_code != 0
+
+    def test_run_command_timeout(self):
+        """Test that a command exceeding the max_run_time times out."""
+        command = "sleep 2"  # A command that takes longer than the timeout
+        stdout, stderr, exit_code, _ = Runner.run_command(command, max_run_time=1)
+        assert stdout == ""
+        assert stderr == "Command timed out"
+        assert exit_code == -1

tests/test_UnitTestValidator.py

@@ -25,6 +25,7 @@ def test_extract_error_message_exception_handling(self):
                 test_command="pytest",
                 llm_model="gpt-3",
                 agent_completion=mock_agent_completion,
+                max_run_time=30,
             )
 
             # Simulate agent_completion raising an exception
@@ -53,6 +54,7 @@ def test_run_coverage_with_report_coverage_flag(self):
                 llm_model="gpt-3",
                 agent_completion=MagicMock(),
                 use_report_coverage_feature_flag=True,
+                max_run_time=30,
             )
             with patch.object(
                 Runner, "run_command", return_value=("", "", 0, datetime.datetime.now())
@@ -79,6 +81,7 @@ def test_extract_error_message_with_prompt_builder(self):
                 test_command="pytest",
                 llm_model="gpt-3",
                 agent_completion=mock_agent_completion,
+                max_run_time=30,
             )
 
             mock_response = """
@@ -138,6 +141,7 @@ def test_validate_test_pass_no_coverage_increase_with_prompt(self):
                 test_command="pytest",
                 llm_model="gpt-3",
                 agent_completion=MagicMock(),
+                max_run_time=30,
             )
 
             # Setup initial state
@@ -181,6 +185,7 @@ def test_initial_test_suite_analysis_with_agent_completion(self):
                 test_command="pytest",
                 llm_model="gpt-3",
                 agent_completion=mock_agent_completion,
+                max_run_time=30,
             )
 
             # Mock responses from agent_completion
@@ -222,6 +227,7 @@ def test_post_process_coverage_report_with_report_coverage_flag(self):
                 llm_model="gpt-3",
                 agent_completion=MagicMock(),
                 use_report_coverage_feature_flag=True,
+                max_run_time=30,
             )
             with patch.object(
                 CoverageProcessor,
@@ -246,6 +252,7 @@ def test_post_process_coverage_report_with_diff_coverage(self):
                 llm_model="gpt-3",
                 agent_completion=MagicMock(),
                 diff_coverage=True,
+                max_run_time=30,
             )
             with patch.object(generator, "generate_diff_coverage_report"), patch.object(
                 CoverageProcessor, "process_coverage_report", return_value=([], [], 0.8)
@@ -266,6 +273,7 @@ def test_post_process_coverage_report_without_flags(self):
                 test_command="pytest",
                 llm_model="gpt-3",
                 agent_completion=MagicMock(),
+                max_run_time=30,
             )
             with patch.object(
                 CoverageProcessor, "process_coverage_report", return_value=([], [], 0.7)
@@ -288,6 +296,7 @@ def test_generate_diff_coverage_report_success(self):
                 agent_completion=MagicMock(),
                 diff_coverage=True,
                 comparison_branch="main",
+                max_run_time=30,
             )
             with patch(
                 "cover_agent.UnitTestValidator.diff_cover_main"
@@ -316,6 +325,7 @@ def test_generate_diff_coverage_report_failure(self):
                 agent_completion=MagicMock(),
                 diff_coverage=True,
                 comparison_branch="main",
+                max_run_time=30,
             )
             with patch(
                 "cover_agent.UnitTestValidator.diff_cover_main",

tests/test_main.py

@@ -128,3 +128,25 @@ def test_main_calls_agent_run(self, mock_isfile, mock_parse_args, mock_cover_age
 
         mock_cover_agent.assert_called_once_with(args)
         mock_agent_instance.run.assert_called_once()
+
+    def test_parse_args_with_max_run_time(self):
+        with patch(
+            "sys.argv",
+            [
+                "program.py",
+                "--source-file-path",
+                "test_source.py",
+                "--test-file-path",
+                "test_file.py",
+                "--code-coverage-report-path",
+                "coverage_report.xml",
+                "--test-command",
+                "pytest",
+                "--max-iterations",
+                "10",
+                "--max-run-time",
+                "45",
+            ],
+        ):
+            args = parse_args()
+            assert args.max_run_time == 45