Add missing grade_cells before autograding (jupyter#1770)

tuncbkose · shreve · commit 1ab41438a0a4 · 2024-07-02T11:13:33.000-04:00
diff --git a/nbgrader/preprocessors/overwritecells.py b/nbgrader/preprocessors/overwritecells.py
@@ -3,14 +3,84 @@
 from .. import utils
 from ..api import Gradebook, MissingEntry
 from . import NbGraderPreprocessor
+from ..nbgraderformat import MetadataValidator
 from nbconvert.exporters.exporter import ResourcesDict
 from nbformat.notebooknode import NotebookNode
+from traitlets import Bool, Unicode
 from typing import Tuple, Any
+from textwrap import dedent
 
 
 class OverwriteCells(NbGraderPreprocessor):
     """A preprocessor to overwrite information about grade and solution cells."""
 
+    add_missing_cells = Bool(
+        False,
+        help=dedent(
+            """
+            Whether or not missing grade_cells should be added back
+            to the notebooks being graded.
+            """
+        ),
+    ).tag(config=True)
+
+    missing_cell_notification = Unicode(
+        "This cell (id:{cell_id}) was missing from the submission. " +
+        "It was added back by nbgrader.\n\n",  # Markdown requires two newlines
+        help=dedent(
+            """
+            A text to add at the beginning of every missing cell re-added to the notebook during autograding.
+            """
+        )
+    ).tag(config=True)
+
+    def missing_cell_transform(self, source_cell, max_score, is_solution=False, is_task=False):
+        """
+        Converts source_cell obtained from Gradebook into a cell that can be added to the notebook.
+        It is assumed that the cell is a grade_cell (unless is_task=True)
+        """
+
+        missing_cell_notification = self.missing_cell_notification.format(cell_id=source_cell.name)
+
+        cell = {
+            "cell_type": source_cell.cell_type,
+            "metadata": {
+                "deletable": False,
+                "editable": False,
+                "nbgrader": {
+                    "grade": True,
+                    "grade_id": source_cell.name,
+                    "locked": source_cell.locked,
+                    "checksum": source_cell.checksum,
+                    "cell_type": source_cell.cell_type,
+                    "points": max_score,
+                    "solution": False
+                }
+            },
+            "source": missing_cell_notification + source_cell.source
+        }
+
+        # Code cell format is slightly different
+        if cell["cell_type"] == "code":
+            cell["execution_count"] = None
+            cell["outputs"] = []
+            cell["source"] = "# " + cell["source"]  # make the notification we add a comment
+
+        # some grade cells are editable (manually graded answers)
+        if is_solution:
+            del cell["metadata"]["editable"]
+            cell["metadata"]["nbgrader"]["solution"] = True
+        # task cells are also a bit different
+        elif is_task:
+            cell["metadata"]["nbgrader"]["grade"] = False
+            cell["metadata"]["nbgrader"]["task"] = True
+            # this is when task cells were added so metadata validation should start from here
+            cell["metadata"]["nbgrader"]["schema_version"] = 3
+
+        cell = NotebookNode(cell)
+        cell = MetadataValidator().upgrade_cell_metadata(cell)
+        return cell
+
     def preprocess(self, nb: NotebookNode, resources: ResourcesDict) -> Tuple[NotebookNode, ResourcesDict]:
         # pull information from the resources
         self.notebook_id = resources['nbgrader']['notebook']
@@ -22,6 +92,77 @@ def preprocess(self, nb: NotebookNode, resources: ResourcesDict) -> Tuple[Notebo
 
         with self.gradebook:
             nb, resources = super(OverwriteCells, self).preprocess(nb, resources)
+            if self.add_missing_cells:
+                nb, resources = self.add_missing_grade_cells(nb, resources)
+                nb, resources = self.add_missing_task_cells(nb, resources)
+
+        return nb, resources
+
+    def add_missing_grade_cells(self, nb: NotebookNode, resources: ResourcesDict) -> Tuple[NotebookNode, ResourcesDict]:
+        """
+        Add missing grade cells back to the notebook.
+        If missing, find the previous solution/grade cell, and add the current cell after it.
+        It is assumed such a cell exists because
+        presumably the grade_cell exists to grade some work in the solution cell.
+        """
+        source_nb = self.gradebook.find_notebook(self.notebook_id, self.assignment_id)
+        source_cells = source_nb.source_cells
+        source_cell_ids = [cell.name for cell in source_cells]
+        grade_cells = {cell.name: cell for cell in source_nb.grade_cells}
+        solution_cell_ids = [cell.name for cell in source_nb.solution_cells]
+
+        # track indices of solution and grade cells in the submitted notebook
+        submitted_cell_idxs = dict()
+        for idx, cell in enumerate(nb.cells):
+            if utils.is_grade(cell) or utils.is_solution(cell):
+                submitted_cell_idxs[cell.metadata.nbgrader["grade_id"]] = idx
+
+        # Every time we add a cell, the idxs above get shifted
+        # We could process the notebook backwards, but that makes adding the cells in the right place more difficult
+        # So we keep track of how many we have added so far
+        added_count = 0
+
+        for grade_cell_id, grade_cell in grade_cells.items():
+            # If missing, find the previous solution/grade cell, and add the current cell after it.
+            if grade_cell_id not in submitted_cell_idxs:
+                self.log.warning(f"Missing grade cell {grade_cell_id} encountered, adding to notebook")
+                source_cell_idx = source_cell_ids.index(grade_cell_id)
+                cell_to_add = source_cells[source_cell_idx]
+                cell_to_add = self.missing_cell_transform(cell_to_add, grade_cell.max_score,
+                                                          is_solution=grade_cell_id in solution_cell_ids)
+                # First cell was deleted, add it to start
+                if source_cell_idx == 0:
+                    nb.cells.insert(0, cell_to_add)
+                    submitted_cell_idxs[grade_cell_id] = 0
+                # Deleted cell is not the first, add it after the previous solution/grade cell
+                else:
+                    prev_cell_id = source_cell_ids[source_cell_idx - 1]
+                    prev_cell_idx = submitted_cell_idxs[prev_cell_id] + added_count
+                    nb.cells.insert(prev_cell_idx + 1, cell_to_add)  # +1 to add it after
+                    submitted_cell_idxs[grade_cell_id] = submitted_cell_idxs[prev_cell_id]
+
+                # If the cell we just added is followed by other missing cells, we need to know its index in the nb
+                # However, no need to add `added_count` to avoid double-counting
+
+                added_count += 1  # shift idxs
+
+        return nb, resources
+
+    def add_missing_task_cells(self, nb: NotebookNode, resources: ResourcesDict) -> Tuple[NotebookNode, ResourcesDict]:
+        """
+        Add missing task cells back to the notebook.
+        We can't figure out their original location, so they are added at the end, in their original order.
+        """
+        source_nb = self.gradebook.find_notebook(self.notebook_id, self.assignment_id)
+        source_cells = source_nb.source_cells
+        source_cell_ids = [cell.name for cell in source_cells]
+        submitted_ids = [cell["metadata"]["nbgrader"]["grade_id"] for cell in nb.cells if
+                         "nbgrader" in cell["metadata"]]
+        for task_cell in source_nb.task_cells:
+            if task_cell.name not in submitted_ids:
+                cell_to_add = source_cells[source_cell_ids.index(task_cell.name)]
+                cell_to_add = self.missing_cell_transform(cell_to_add, task_cell.max_score, is_task=True)
+                nb.cells.append(cell_to_add)
 
         return nb, resources
 
diff --git a/nbgrader/tests/preprocessors/test_overwritecells.py b/nbgrader/tests/preprocessors/test_overwritecells.py
@@ -1,14 +1,14 @@
 import pytest
 
-from nbformat.v4 import new_notebook
+from nbformat.v4 import new_notebook, new_markdown_cell
 
 from ...preprocessors import SaveCells, OverwriteCells
 from ...api import Gradebook
 from ...utils import compute_checksum
 from .base import BaseTestPreprocessor
 from .. import (
     create_grade_cell, create_solution_cell, create_grade_and_solution_cell,
-    create_locked_cell)
+    create_locked_cell, create_task_cell)
 
 
 @pytest.fixture
@@ -23,6 +23,7 @@ def gradebook(request, db):
 
     def fin():
         gb.close()
+
     request.addfinalizer(fin)
 
     return gb
@@ -197,7 +198,7 @@ def test_overwrite_locked_checksum(self, preprocessors, resources):
 
         assert cell.metadata.nbgrader["checksum"] == compute_checksum(cell)
 
-    def test_nonexistant_grade_id(self, preprocessors, resources):
+    def test_nonexistent_grade_id(self, preprocessors, resources):
         """Are cells not in the database ignored?"""
         cell = create_grade_cell("", "code", "", 1)
         cell.metadata.nbgrader['grade'] = False
@@ -215,3 +216,52 @@ def test_nonexistant_grade_id(self, preprocessors, resources):
         nb, resources = preprocessors[1].preprocess(nb, resources)
         assert 'grade_id' not in cell.metadata.nbgrader
 
+    # Tests for adding missing cells back
+    def test_add_missing_cells(self, preprocessors, resources):
+        """
+        Note: This test will produce warnings (from OverwriteCells preprocessor) by default.
+        Current implementation of adding missing cells should:
+        - add missing cells right after the previous grade/solution cell, as the best approximation of their location
+        - add task cells at the end (because we can't detect their location in the notebook), in order of appearance
+        """
+        
+        cells = [
+            create_solution_cell("Code assignment", "code", "code_solution"),
+            create_grade_cell("some tests", "code", "code_test1", 1),
+            create_grade_cell("more tests", "code", "code_test2", 1),
+            new_markdown_cell("some description"),
+            create_grade_and_solution_cell("write answer here", "markdown", "md_manually_graded1", 1),
+            create_grade_and_solution_cell("write answer here", "markdown", "md_manually_graded2", 1),
+            new_markdown_cell("some description"),
+            create_task_cell("some task description", "markdown", "task_cell1", 1),
+            new_markdown_cell("some description"),
+            create_task_cell("some task description", "markdown", "task_cell2", 1),
+        ]
+        # Add checksums to suppress warning
+        nbgrader_cells = [0, 1, 2, 4, 5, 7, 9]
+        for idx, cell in enumerate(cells):
+            if idx in nbgrader_cells:
+                cell.metadata.nbgrader["checksum"] = compute_checksum(cell)
+
+        expected_order = [0, 1, 2, 4, 5, 3, 6, 8, 7, 9]
+        expected = [cells[i].metadata.nbgrader["grade_id"] if "nbgrader" in cells[i].metadata else "markdown" for i in expected_order]
+
+        nb = new_notebook()
+        nb.cells = cells
+
+        # save to database
+        nb, resources = preprocessors[0].preprocess(nb, resources)
+
+        # remove grade/task cells to test their restoration
+        nb.cells.pop(9)
+        nb.cells.pop(7)
+        nb.cells.pop(5)
+        nb.cells.pop(4)
+        nb.cells.pop(2)
+        nb.cells.pop(1)
+
+        # restore
+        preprocessors[1].add_missing_cells = True
+        nb, resources = preprocessors[1].preprocess(nb, resources)
+        result = [cell["metadata"]["nbgrader"]["grade_id"] if "nbgrader" in cell["metadata"] else "markdown" for cell in nb.cells]
+        assert expected == result
