Feature-329: Refactor run_tests_all.py to work within pytest framework (#330)

* Minor initial changes

* Add initial pytest stuff

* Add README.md for configuration.toml file

* Fix pytest.ini

* Improving Docker test script

* Fix cover-agent-full-repo

* Adjust CI pipeline for E2E tests

* Some code reformatting

Author: andsimakov

.github/workflows/ci_pipeline.yml

@@ -153,13 +153,72 @@ jobs:
         name: cover-agent-${{ matrix.os }}
         path: dist/cover-agent*
 
+  e2e-test:
+    # This job will run after building binary because it needs the binary to run the E2E tests in Docker
+    needs: build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.12'
+
+      # Download the Ubuntu binary
+      - name: Download Ubuntu executable
+        uses: actions/download-artifact@v4
+        with:
+          name: cover-agent-ubuntu-22.04
+          path: dist
+
+      # Make the binary executable
+      - name: Make binary executable
+        run: chmod +x dist/cover-agent
+
+      # Cache and install dependencies
+      - name: Cache Poetry dependencies
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/pypoetry
+            ~/.cache/pip
+          key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-poetry-
+
+      - name: Install Poetry
+        run: pip install poetry
+      - name: Install dependencies using Poetry
+        run: poetry install
+
+      # Run the E2E Docker tests using the Makefile target `e2e-test`
+      - name: Run E2E Docker tests
+        env:
+          OPENAI_API_KEY: "This_is_a_fake_API_key"
+        run: |
+          make e2e-test 2>&1 | tee e2e_test_full.log
+
+      # Upload E2E test reports
+      - name: Upload E2E Docker test report
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: e2e-test-reports
+          path: |
+            testLog_e2e.xml
+            e2e_test_full.log
+
   release:
     # This job will run only if the following conditions are met:
     # - The event is a 'push' to the 'main' branch
     # - The 'version.txt' file has changed
-    # - There are relevant changes outside of 'README.md' and the 'docs/' folder
-    needs: [build, changes]
-    if: github.event_name == 'push' && github.ref == 'refs/heads/main' && needs.changes.outputs.version_changed == 'true' && needs.changes.outputs.relevant_changes == 'true'
+    # - There are relevant changes outside 'README.md' and the 'docs/' folder
+    needs: [build, changes, e2e-test]
+    if: >
+      github.event_name == 'push' && 
+      github.ref == 'refs/heads/main' &&
+      needs.changes.outputs.version_changed == 'true' &&
+      needs.changes.outputs.relevant_changes == 'true'
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2

Makefile

@@ -7,7 +7,21 @@ TOML_FILES=$(shell find cover_agent/settings -name "*.toml" | sed 's/.*/-\-add-d
 
 # Run unit tests with Pytest
 test:
-	poetry run pytest --junitxml=testLog.xml --cov=cover_agent --cov-report=xml:cobertura.xml --cov-report=term --cov-fail-under=65 --log-cli-level=INFO
+	poetry run pytest \
+		-m "not e2e_docker" \
+		--junitxml=testLog.xml \
+		--cov=cover_agent \
+		--cov-report=xml:cobertura.xml \
+		--cov-report=term \
+		--cov-fail-under=65
+		--log-cli-level=INFO
+
+e2e-test:
+	poetry run pytest \
+		-m "e2e_docker" \
+		--capture=no \
+		--junitxml=testLog_e2e.xml \
+		--log-cli-level=INFO
 
 # Use Python Black to format python files
 format:

cover_agent/CoverAgent.py

@@ -2,20 +2,21 @@
 import os
 import shutil
 import sys
-import wandb
 
 from typing import Optional
 
-from cover_agent.CustomLogger import CustomLogger
-from cover_agent.UnitTestGenerator import UnitTestGenerator
-from cover_agent.UnitTestValidator import UnitTestValidator
-from cover_agent.UnitTestDB import UnitTestDB
-from cover_agent.AICaller import AICaller
+import wandb
+
 from cover_agent.AgentCompletionABC import AgentCompletionABC
-from cover_agent.DefaultAgentCompletion import DefaultAgentCompletion
 from cover_agent.ai_caller_replay import AICallerReplay
+from cover_agent.AICaller import AICaller
+from cover_agent.CustomLogger import CustomLogger
+from cover_agent.DefaultAgentCompletion import DefaultAgentCompletion
 from cover_agent.record_replay_manager import RecordReplayManager
 from cover_agent.settings.config_schema import CoverAgentConfig
+from cover_agent.UnitTestDB import UnitTestDB
+from cover_agent.UnitTestGenerator import UnitTestGenerator
+from cover_agent.UnitTestValidator import UnitTestValidator
 
 
 class CoverAgent:
@@ -381,6 +382,7 @@ def run(self):
         failed_test_runs, language, test_framework, coverage_report = self.init()
 
         while iteration_count < self.config.max_iterations:
+            self.logger.info(f"Iteration {iteration_count + 1} of {self.config.max_iterations}.")
             self.generate_and_validate_tests(failed_test_runs, language, test_framework, coverage_report)
 
             failed_test_runs, language, test_framework, coverage_report, target_reached = self.check_iteration_progress()

cover_agent/UnitTestValidator.py

@@ -198,9 +198,9 @@ def initial_test_suite_analysis(self):
             None
         """
         try:
+            settings = get_settings().get("default")
             test_headers_indentation = None
-            # TODO: Move to configuration.toml?
-            allowed_attempts = 3
+            allowed_attempts = settings.get("test_headers_indentation_attempts", 3)
             counter_attempts = 0
             while (
                 test_headers_indentation is None and counter_attempts < allowed_attempts
@@ -233,8 +233,6 @@ def initial_test_suite_analysis(self):
 
             relevant_line_number_to_insert_tests_after = None
             relevant_line_number_to_insert_imports_after = None
-            # TODO: Move to configuration.toml?
-            allowed_attempts = 3
             counter_attempts = 0
             while (
                 not relevant_line_number_to_insert_tests_after

cover_agent/__main__.py

@@ -1,44 +1,39 @@
 import asyncio
 import copy
-import os
+
 from cover_agent.AICaller import AICaller
-from cover_agent.utils import parse_args_full_repo, find_test_files
 from cover_agent.CoverAgent import CoverAgent
 from cover_agent.lsp_logic.ContextHelper import ContextHelper
+from cover_agent.settings.config_loader import get_settings
+from cover_agent.settings.config_schema import CoverAgentConfig
+from cover_agent.utils import find_test_files, parse_args_full_repo
 
 
 async def run():
-    args = parse_args_full_repo()
+    settings = get_settings().get("default")
+    args = parse_args_full_repo(settings)
 
     if args.project_language == "python":
         context_helper = ContextHelper(args)
     else:
-        raise NotImplementedError(
-            "Unsupported language: {}".format(args.project_language)
-        )
+        raise NotImplementedError("Unsupported language: {}".format(args.project_language))
 
     # scan the project directory for test files
     test_files = find_test_files(args)
-    print(
-        "============\nTest files to be extended:\n"
-        + "".join(f"{f}\n============\n" for f in test_files)
-    )
+    print("============\nTest files to be extended:\n" + "".join(f"{f}\n============\n" for f in test_files))
 
     # start the language server
     async with context_helper.start_server():
         print("LSP server initialized.")
 
-        ai_caller = AICaller(model=args.model)
+        generate_log_files = not args.suppress_log_files
+        ai_caller = AICaller(model=args.model, generate_log_files=generate_log_files)
 
         # main loop for analyzing test files
         for test_file in test_files:
             # Find the context files for the test file
             context_files = await context_helper.find_test_file_context(test_file)
-            print(
-                "Context files for test file '{}':\n{}".format(
-                    test_file, "".join(f"{f}\n" for f in context_files)
-                )
-            )
+            print("Context files for test file '{}':\n{}".format(test_file, "".join(f"{f}\n" for f in context_files)))
 
             # Analyze the test file against the context files
             print("\nAnalyzing test file against context files...")
@@ -54,7 +49,9 @@ async def run():
                     args_copy.test_command_dir = args.project_root
                     args_copy.test_file_path = test_file
                     args_copy.included_files = context_files_include
-                    agent = CoverAgent(args_copy)
+
+                    config = CoverAgentConfig.from_cli_args_with_defaults(args_copy)
+                    agent = CoverAgent(config)
                     agent.run()
                 except Exception as e:
                     print(f"Error running CoverAgent for test file '{test_file}': {e}")

cover_agent/main_full_repo.py

@@ -0,0 +1,48 @@
+# [DRAFT] Configuration Parameters
+
+Explanation of parameters from the `configuration.toml` file.
+
+## [default]
+
+### LLM Configuration
+- `model`: The LLM model to use for test generation (default: `gpt-4o-2024-11-20`)
+- `api_base`: Base URL for API calls, used for local LLM servers (default: `http://localhost:11434`)
+- `model_retries`: Number of retry attempts for failed LLM calls (default: `3`)
+
+### Coverage Settings
+- `desired_coverage`: Target code coverage percentage to achieve (default: `70`)
+- `coverage_type`: Type of coverage report to generate (default: `cobertura`)
+
+### Execution Limits
+- `max_iterations`: Maximum number of test generation iterations (default: `3`)
+- `max_run_time_sec`: Maximum runtime in seconds for each test generation attempt (default: `30`)
+- `max_tests_per_run`: Maximum number of tests to generate per run (default: `4`)
+- `allowed_initial_test_analysis_attempts`: Number of attempts for initial test analysis (default: `3`)
+- `run_tests_multiple_times`: Number of times to run each test for consistency (default: `1`)
+
+### File Paths
+- `log_file_path`: Path to the main log file and its name (default: `run.log`)
+- `log_db_path`: Path to the SQLite database for logging and its name (default: `cover_agent_unit_test_runs.db`)
+- `report_filepath`: Path to the HTML test results report and its name (default: `test_results.html`)
+- `responses_folder`: Directory for storing LLM responses (default: `stored_responses`)
+
+### Docker Settings
+- `cover_agent_host_folder`: Host machine folder for cover-agent (default: `dist/cover-agent`)
+- `cover_agent_container_folder`: Container folder for cover-agent (default: `/usr/local/bin/cover-agent`)
+- `docker_hash_display_length`: Length of displayed Docker hash (default:`12`)
+- `record_replay_hash_display_length`: Length of displayed record/replay hash (default: `12`)
+
+### Git Settings
+- `branch`: Git branch to use (default: `main`)
+
+### Fuzzy Lookup Settings
+- `fuzzy_lookup_threshold`: Threshold for fuzzy matching (default:`95`)
+- `fuzzy_lookup_prefix_length`: Prefix length for fuzzy lookup (default: `1000`)
+- `fuzzy_lookup_best_ratio`: Best ratio for fuzzy matching (default: `0`)
+
+## [include_files]
+- `limit_tokens`: Whether to limit tokens for included files (default: `true`)
+- `max_tokens`: Maximum tokens allowed for included files (default: `20000`)
+
+## [tests]
+- `max_allowed_runtime_seconds`: Maximum allowed runtime for tests in seconds (default: `30`)

cover_agent/settings/README.md

@@ -17,6 +17,7 @@ class CoverageType(Enum):
         COBERTURA: Represents the Cobertura coverage report format.
         JACOCO: Represents the JaCoCo coverage report format.
     """
+
     LCOV = "lcov"
     COBERTURA = "cobertura"
     JACOCO = "jacoco"
@@ -57,6 +58,7 @@ class CoverAgentConfig:
         record_mode (bool): Enable LLM responses record mode for tests.
         test_command_original (str): Original test command before any modifications.
     """
+
     source_file_path: str
     test_file_path: str
     project_root: str
@@ -82,6 +84,9 @@ class CoverAgentConfig:
     run_each_test_separately: bool
     record_mode: bool
     suppress_log_files: bool
+    max_test_files_allowed_to_analyze: int
+    look_for_oldest_unchanged_test_file: bool
+    project_language: str
     test_command_original: Optional[str] = None
 
     @classmethod
@@ -122,6 +127,9 @@ def from_cli_args(cls, args: argparse.Namespace) -> "CoverAgentConfig":
             run_each_test_separately=args.run_each_test_separately,
             record_mode=args.record_mode,
             suppress_log_files=args.suppress_log_files,
+            max_test_files_allowed_to_analyze=args.max_test_files_allowed_to_analyze,
+            look_for_oldest_unchanged_test_file=args.look_for_oldest_unchanged_test_file,
+            project_language=args.project_language,
         )
 
     @classmethod
@@ -165,6 +173,9 @@ def from_cli_args_with_defaults(cls, args: argparse.Namespace) -> "CoverAgentCon
             "run_each_test_separately": default_config.get("run_each_test_separately"),
             "record_mode": default_config.get("record_mode"),
             "suppress_log_files": default_config.get("suppress_log_files"),
+            "max_test_files_allowed_to_analyze": default_config.get("max_test_files_allowed_to_analyze"),
+            "look_for_oldest_unchanged_test_file": default_config.get("look_for_oldest_unchanged_test_file"),
+            "project_language": default_config.get("project_language"),
         }
 
         # CLI overrides default settings

cover_agent/settings/config_schema.py

@@ -1,13 +1,18 @@
 [default]
 model = "gpt-4o-2024-11-20"
+model_full_repo = "bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0"
 desired_coverage = 70
+desired_coverage_full_repo = 100
 max_iterations = 3
+max_test_files_allowed_to_analyze = 20
 api_base = "http://localhost:11434"
 max_run_time_sec = 30
 max_tests_per_run = 4
+allowed_initial_test_analysis_attempts = 3
 model_retries = 3
 run_tests_multiple_times = 1
 branch = "main"
+project_language = "python"
 coverage_type = "cobertura"
 log_file_path = "run.log"
 log_db_path = "cover_agent_unit_test_runs.db"