diff --git a/magi/common/gradescope.py b/magi/common/gradescope.py index fbcf742..991bf3d 100644 --- a/magi/common/gradescope.py +++ b/magi/common/gradescope.py @@ -23,18 +23,22 @@ class TestCase: # tags: list = field(default_factory=list) # extra_data:str = "" - def fail_test(self, msg: str): - self.output += "\n" + msg + def fail(self, msg: str=""): + if msg: + self.output += "\n" + msg self.score = 0 self.status = "failed" - def pass_test(self, msg: str): - self.output += "\n" + msg + def succ(self, msg: str=""): + if msg: + self.output += "\n" + msg self.score = self.max_score self.status = "passed" def add_output_msg(self, msg: str): - self.output += "\n" + msg + if self.output: + self.output += "\n" + self.output += msg @dataclass diff --git a/magi/managers/test_manager.py b/magi/managers/test_manager.py index ceee206..0e62205 100644 --- a/magi/managers/test_manager.py +++ b/magi/managers/test_manager.py @@ -5,6 +5,7 @@ from typing import Optional from magi.common.gradescope import TestCase, Result +from magi.utils.serialization import serialize logging = logging.getLogger("TestManager") @@ -19,10 +20,16 @@ def __init__(self): self.test_cases_by_name = {} self.anonymous_counter: int = 0 self.all_failed: bool = False # if True, the total score and Testcases will be zero - - -status = Status() - + + def reset(self): + self.score = 0 + self.execution_time = 0 + self.output = "" + self.extra_data = {} + self.test_cases = [] + self.test_cases_by_name = {} + self.anonymous_counter = 0 + self.all_failed = False def reset(): """ @@ -93,13 +100,17 @@ def output_result(result_path: Optional[str | Path] = None) -> None: from magi.managers import InfoManager result_path = InfoManager.Directories.RESULT_JSON_PATH - from magi.utils.serialization import serialize result = Result() result.output += status.output + if not status.test_cases: + logging.warning("No test cases were added") + status.test_cases.append(TestCase(name="No test cases were executed", score=0, output="")) + for test in status.test_cases: result.tests.append(test) if status.all_failed: result.score = 0 + serialize(result, result_path) diff --git a/plugins/c_cpp/c_cpp.py b/plugins/c_cpp/c_cpp.py index 9aead24..d38c9d7 100644 --- a/plugins/c_cpp/c_cpp.py +++ b/plugins/c_cpp/c_cpp.py @@ -15,13 +15,11 @@ def before_grading(self): for dir in Config.ensure_dirs: os.makedirs(Directories.WORK_DIR / dir, exist_ok=True) - if Config.compile_method == CompileMethod.INSTRUCTOR_MAKE: - p = subprocess.run(["make"], cwd=Directories.WORK_DIR, capture_output=True) - elif Config.compile_method == CompileMethod.STUDENT_MAKE: - p = run("make", cwd=Directories.WORK_DIR, capture_output=True) - elif Config.compile_method == CompileMethod.AUTO_CMAKE: + if Config.compile_method == CompileMethod.AUTO_CMAKE: p = subprocess.run(["cmake", "."], cwd=Directories.WORK_DIR, capture_output=True) - + else: + p = run("make", cwd=Directories.WORK_DIR, capture_output=True) + if p.returncode != 0: TestManager.fail_all(f"Compilation failed with error: {p.stderr.decode()}\n {p.stdout.decode()}") @@ -33,3 +31,9 @@ def before_generating(self): shutil.copy(Config.makefile, Directories.WORK_DIR / "Makefile") if Config.provide_student_makefile: shutil.copy(Config.makefile, Directories.OUTPUT_DIR / "dist" / "Makefile") + if Config.create_file_structure: + directories = ["include", "src", "test", "obj", "lib"] + if Config.create_file_structure: + for dir in directories: + os.makedirs(Directories.WORK_DIR / dir, exist_ok=True) + \ No newline at end of file diff --git a/plugins/c_cpp/config.py b/plugins/c_cpp/config.py index 5c634c4..f7bb8cd 100644 --- a/plugins/c_cpp/config.py +++ b/plugins/c_cpp/config.py @@ -24,4 +24,8 @@ class Config: makefile: str = field(default="plugins/c_cpp/Makefile", metadata={"excluded_from_ui": True, "file_editor": "plugins/c_cpp/Makefile"}) + create_file_structure: bool = field(default=True, + metadata={"help": "Create a file structure for C/C++ projects", + "half_width": True}) + ensure_dirs: list[str] = field(default_factory=lambda: ["include", "src", "test", "obj", "lib"]) diff --git a/plugins/gtest/grader.py b/plugins/gtest/grader.py index 5eba63c..1c44e2a 100644 --- a/plugins/gtest/grader.py +++ b/plugins/gtest/grader.py @@ -10,7 +10,9 @@ from magi.common import gradescope from magi.common.gradescope import TestCase from magi.managers import TestManager +from magi.managers.info_manager import Directories from magi.utils import file_utils +from magi.utils.code_runner import Popen @dataclass @@ -77,6 +79,8 @@ def generate_overall_stats(self): print("Errors: " + str(self.errors)) print("Disabled: " + str(self.disabled)) + def all_tests_passed(self): + return self.tests == self.tests - (self.failures + self.disabled + self.errors) def write_failed_test(fname: str, testname: str, points: str) -> None: """Generates a generic failure XML document which can be parsed in the same way that succeeding @@ -110,14 +114,13 @@ def grade_all(test_file_name: str) -> None: :param str test_file_name: The path to the yaml file describing the tests to execute. """ - from magi.utils.code_runner import Popen - from magi.managers.info_manager import Directories # First, we're going to read in the test layout from the user defined YAML file with open(test_file_name, 'r') as file: tests = yaml.load(file, Loader=yaml.FullLoader) TEMP_DIR = Directories.WORK_DIR / "gtest_temp" - os.makedirs(TEMP_DIR, exist_ok=True) + file_utils.reset_dir(TEMP_DIR) + os.chmod(TEMP_DIR, 0o777) for test in tests['tests']: # define the output name for the gtest xml file, as gtest can only export in xml, no python API yet @@ -127,41 +130,61 @@ def grade_all(test_file_name: str) -> None: logging.debug(f"Running: {test['name']}") # In case we are running the autograder again, we want to remove any existing XML files which may be present - # if os.path.exists(out_name): - # os.remove(out_name) file_utils.remove(out_name) # Run the individual gtest using the following console command (subprocess is how you can run system commands from python) + out = "" + err = "" gtest_filter = test['class'] + "." + test['name'] gtest_output = "xml:" + str(out_name) - try: - p = Popen(["./" + test['file'], f"--gtest_output={gtest_output}", f"--gtest_filter={gtest_filter}"], - stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=Directories.WORK_DIR) + test_case = TestCase(name=gtest_filter, max_score=test['points']) + test_case.visibility = gradescope.Visibility.VISIBLE if 'visible' not in test or test[ + 'visible'] else gradescope.Visibility.AFTER_PUBLISHED + + try: + command = ["./" + test['file'], f"--gtest_output={gtest_output}", f"--gtest_filter={gtest_filter}"] + p = Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=Directories.WORK_DIR) # Get stdout and stderr out, err = p.communicate() # gtest outputs 0 if test succeeds, 1 otherwise, 124 if timeout logging.debug(f"Test {test['name']} returned {p.returncode}") if p.returncode != 0: - logging.warning(f"Test {test['name']} returned {p.returncode}") + logging.warning(f"Test {test['name']} returned {p.returncode} :{out}\n {err}") + except Exception as e: logging.error(f"Error running test {test['name']}: {e}", exc_info=True) - - # If the xml fails to generate, the test fails to execute (possibly due to segfault) + test_case.fail(str(e)) + + def fail_test(): + message = "" + if out: + message += str(out) + "\n" + if err: + message += str(err) + test_case.fail(message) + TestManager.add_test(test_case) + if not os.path.exists(out_name): - logging.warning(f"Test {test['name']} failed to execute") - # Write a generic failed XML file so that we can treat it the same as other tests with one function - write_failed_test(out_name, test['name'], test['points']) + fail_test() + continue + xml_test_suite = XmlTestSuite(str(out_name)) xml_test_cases = xml_test_suite.gather_data() - for xml_test_case in xml_test_cases: - test_case = TestCase(name=xml_test_case.name, max_score=test['points']) - test_case.visibility = gradescope.Visibility.VISIBLE if 'visible' not in test or test[ - 'visible'] else gradescope.Visibility.AFTER_PUBLISHED - if xml_test_case.status == "error": - test_case.fail_test(xml_test_case.failed_msg) - else: - test_case.pass_test(xml_test_case.failed_msg) - TestManager.add_test(test_case) + if not xml_test_cases: + fail_test() + continue + + xml_test_case = xml_test_cases[0] + + if xml_test_suite.all_tests_passed(): + test_case.succ() + else: + test_case.fail(xml_test_case.failed_msg) + + TestManager.add_test(test_case) + + +