From e4d8b4f8454f3356a099908b9080f53513078569 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tun=C3=A7=20Ba=C5=9Far=20K=C3=B6se?= Date: Tue, 28 Mar 2023 11:56:28 +0300 Subject: [PATCH] Add missing grade_cells before autograding --- nbgrader/preprocessors/overwritecells.py | 141 ++++++++++++++++++ .../preprocessors/test_overwritecells.py | 56 ++++++- 2 files changed, 194 insertions(+), 3 deletions(-) diff --git a/nbgrader/preprocessors/overwritecells.py b/nbgrader/preprocessors/overwritecells.py index 7a4a628b8..f44f046ea 100644 --- a/nbgrader/preprocessors/overwritecells.py +++ b/nbgrader/preprocessors/overwritecells.py @@ -3,14 +3,84 @@ from .. import utils from ..api import Gradebook, MissingEntry from . import NbGraderPreprocessor +from ..nbgraderformat import MetadataValidator from nbconvert.exporters.exporter import ResourcesDict from nbformat.notebooknode import NotebookNode +from traitlets import Bool, Unicode from typing import Tuple, Any +from textwrap import dedent class OverwriteCells(NbGraderPreprocessor): """A preprocessor to overwrite information about grade and solution cells.""" + add_missing_cells = Bool( + False, + help=dedent( + """ + Whether or not missing grade_cells should be added back + to the notebooks being graded. + """ + ), + ).tag(config=True) + + missing_cell_notification = Unicode( + "This cell (id:{cell_id}) was missing from the submission. " + + "It was added back by nbgrader.\n\n", # Markdown requires two newlines + help=dedent( + """ + A text to add at the beginning of every missing cell re-added to the notebook during autograding. + """ + ) + ).tag(config=True) + + def missing_cell_transform(self, source_cell, max_score, is_solution=False, is_task=False): + """ + Converts source_cell obtained from Gradebook into a cell that can be added to the notebook. + It is assumed that the cell is a grade_cell (unless is_task=True) + """ + + missing_cell_notification = self.missing_cell_notification.format(cell_id=source_cell.name) + + cell = { + "cell_type": source_cell.cell_type, + "metadata": { + "deletable": False, + "editable": False, + "nbgrader": { + "grade": True, + "grade_id": source_cell.name, + "locked": source_cell.locked, + "checksum": source_cell.checksum, + "cell_type": source_cell.cell_type, + "points": max_score, + "solution": False + } + }, + "source": missing_cell_notification + source_cell.source + } + + # Code cell format is slightly different + if cell["cell_type"] == "code": + cell["execution_count"] = None + cell["outputs"] = [] + cell["source"] = "# " + cell["source"] # make the notification we add a comment + + # some grade cells are editable (manually graded answers) + if is_solution: + del cell["metadata"]["editable"] + cell["metadata"]["nbgrader"]["solution"] = True + # task cells are also a bit different + elif is_task: + cell["metadata"]["nbgrader"]["grade"] = False + cell["metadata"]["nbgrader"]["task"] = True + # this is when task cells were added so metadata validation should start from here + cell["metadata"]["nbgrader"]["schema_version"] = 3 + + cell = NotebookNode(cell) + cell = MetadataValidator().upgrade_cell_metadata(cell) + return cell + def preprocess(self, nb: NotebookNode, resources: ResourcesDict) -> Tuple[NotebookNode, ResourcesDict]: # pull information from the resources self.notebook_id = resources['nbgrader']['notebook'] @@ -22,6 +92,77 @@ def preprocess(self, nb: NotebookNode, resources: ResourcesDict) -> Tuple[Notebo with self.gradebook: nb, resources = super(OverwriteCells, self).preprocess(nb, resources) + if self.add_missing_cells: + nb, resources = self.add_missing_grade_cells(nb, resources) + nb, resources = self.add_missing_task_cells(nb, resources) + + return nb, resources + + def add_missing_grade_cells(self, nb: NotebookNode, resources: ResourcesDict) -> Tuple[NotebookNode, ResourcesDict]: + """ + Add missing grade cells back to the notebook. + If missing, find the previous solution/grade cell, and add the current cell after it. + It is assumed such a cell exists because + presumably the grade_cell exists to grade some work in the solution cell. + """ + source_nb = self.gradebook.find_notebook(self.notebook_id, self.assignment_id) + source_cells = source_nb.source_cells + source_cell_ids = [cell.name for cell in source_cells] + grade_cells = {cell.name: cell for cell in source_nb.grade_cells} + solution_cell_ids = [cell.name for cell in source_nb.solution_cells] + + # track indices of solution and grade cells in the submitted notebook + submitted_cell_idxs = dict() + for idx, cell in enumerate(nb.cells): + if utils.is_grade(cell) or utils.is_solution(cell): + submitted_cell_idxs[cell.metadata.nbgrader["grade_id"]] = idx + + # Every time we add a cell, the idxs above get shifted + # We could process the notebook backwards, but that makes adding the cells in the right place more difficult + # So we keep track of how many we have added so far + added_count = 0 + + for grade_cell_id, grade_cell in grade_cells.items(): + # If missing, find the previous solution/grade cell, and add the current cell after it. + if grade_cell_id not in submitted_cell_idxs: + self.log.warning(f"Missing grade cell {grade_cell_id} encountered, adding to notebook") + source_cell_idx = source_cell_ids.index(grade_cell_id) + cell_to_add = source_cells[source_cell_idx] + cell_to_add = self.missing_cell_transform(cell_to_add, grade_cell.max_score, + is_solution=grade_cell_id in solution_cell_ids) + # First cell was deleted, add it to start + if source_cell_idx == 0: + nb.cells.insert(0, cell_to_add) + submitted_cell_idxs[grade_cell_id] = 0 + # Deleted cell is not the first, add it after the previous solution/grade cell + else: + prev_cell_id = source_cell_ids[source_cell_idx - 1] + prev_cell_idx = submitted_cell_idxs[prev_cell_id] + added_count + nb.cells.insert(prev_cell_idx + 1, cell_to_add) # +1 to add it after + submitted_cell_idxs[grade_cell_id] = submitted_cell_idxs[prev_cell_id] + + # If the cell we just added is followed by other missing cells, we need to know its index in the nb + # However, no need to add `added_count` to avoid double-counting + + added_count += 1 # shift idxs + + return nb, resources + + def add_missing_task_cells(self, nb: NotebookNode, resources: ResourcesDict) -> Tuple[NotebookNode, ResourcesDict]: + """ + Add missing task cells back to the notebook. + We can't figure out their original location, so they are added at the end, in their original order. + """ + source_nb = self.gradebook.find_notebook(self.notebook_id, self.assignment_id) + source_cells = source_nb.source_cells + source_cell_ids = [cell.name for cell in source_cells] + submitted_ids = [cell["metadata"]["nbgrader"]["grade_id"] for cell in nb.cells if + "nbgrader" in cell["metadata"]] + for task_cell in source_nb.task_cells: + if task_cell.name not in submitted_ids: + cell_to_add = source_cells[source_cell_ids.index(task_cell.name)] + cell_to_add = self.missing_cell_transform(cell_to_add, task_cell.max_score, is_task=True) + nb.cells.append(cell_to_add) return nb, resources diff --git a/nbgrader/tests/preprocessors/test_overwritecells.py b/nbgrader/tests/preprocessors/test_overwritecells.py index 73284a515..4f768f77c 100644 --- a/nbgrader/tests/preprocessors/test_overwritecells.py +++ b/nbgrader/tests/preprocessors/test_overwritecells.py @@ -1,6 +1,6 @@ import pytest -from nbformat.v4 import new_notebook +from nbformat.v4 import new_notebook, new_markdown_cell from ...preprocessors import SaveCells, OverwriteCells from ...api import Gradebook @@ -8,7 +8,7 @@ from .base import BaseTestPreprocessor from .. import ( create_grade_cell, create_solution_cell, create_grade_and_solution_cell, - create_locked_cell) + create_locked_cell, create_task_cell) @pytest.fixture @@ -23,6 +23,7 @@ def gradebook(request, db): def fin(): gb.close() + request.addfinalizer(fin) return gb @@ -197,7 +198,7 @@ def test_overwrite_locked_checksum(self, preprocessors, resources): assert cell.metadata.nbgrader["checksum"] == compute_checksum(cell) - def test_nonexistant_grade_id(self, preprocessors, resources): + def test_nonexistent_grade_id(self, preprocessors, resources): """Are cells not in the database ignored?""" cell = create_grade_cell("", "code", "", 1) cell.metadata.nbgrader['grade'] = False @@ -215,3 +216,52 @@ def test_nonexistant_grade_id(self, preprocessors, resources): nb, resources = preprocessors[1].preprocess(nb, resources) assert 'grade_id' not in cell.metadata.nbgrader + # Tests for adding missing cells back + def test_add_missing_cells(self, preprocessors, resources): + """ + Note: This test will produce warnings (from OverwriteCells preprocessor) by default. + Current implementation of adding missing cells should: + - add missing cells right after the previous grade/solution cell, as the best approximation of their location + - add task cells at the end (because we can't detect their location in the notebook), in order of appearance + """ + + cells = [ + create_solution_cell("Code assignment", "code", "code_solution"), + create_grade_cell("some tests", "code", "code_test1", 1), + create_grade_cell("more tests", "code", "code_test2", 1), + new_markdown_cell("some description"), + create_grade_and_solution_cell("write answer here", "markdown", "md_manually_graded1", 1), + create_grade_and_solution_cell("write answer here", "markdown", "md_manually_graded2", 1), + new_markdown_cell("some description"), + create_task_cell("some task description", "markdown", "task_cell1", 1), + new_markdown_cell("some description"), + create_task_cell("some task description", "markdown", "task_cell2", 1), + ] + # Add checksums to suppress warning + nbgrader_cells = [0, 1, 2, 4, 5, 7, 9] + for idx, cell in enumerate(cells): + if idx in nbgrader_cells: + cell.metadata.nbgrader["checksum"] = compute_checksum(cell) + + expected_order = [0, 1, 2, 4, 5, 3, 6, 8, 7, 9] + expected = [cells[i].metadata.nbgrader["grade_id"] if "nbgrader" in cells[i].metadata else "markdown" for i in expected_order] + + nb = new_notebook() + nb.cells = cells + + # save to database + nb, resources = preprocessors[0].preprocess(nb, resources) + + # remove grade/task cells to test their restoration + nb.cells.pop(9) + nb.cells.pop(7) + nb.cells.pop(5) + nb.cells.pop(4) + nb.cells.pop(2) + nb.cells.pop(1) + + # restore + preprocessors[1].add_missing_cells = True + nb, resources = preprocessors[1].preprocess(nb, resources) + result = [cell["metadata"]["nbgrader"]["grade_id"] if "nbgrader" in cell["metadata"] else "markdown" for cell in nb.cells] + assert expected == result