Add support for sample cases to judge and protocol; #615

dmoj/graders/base.py

@@ -29,18 +29,28 @@ def terminate_grading(self):
             except OSError:
                 pass
 
-    def _resolve_testcases(self, cfg, batch_no=0):
+    def _resolve_testcases(self, cfg, batch_no=0, kind=TestCase.KIND_NORMAL):
         cases = []
         for case_config in cfg:
+            # Hack for backwards-compatibility: if points are set to 0, this is a
+            # pretest regardless of whatever [kind] was specified to be.
+            real_kind = kind if case_config.points else TestCase.KIND_PRETEST
             if 'batched' in case_config.raw_config:
                 self._batch_counter += 1
-                cases.append(BatchedTestCase(self._batch_counter, case_config, self.problem,
-                                             self._resolve_testcases(case_config['batched'], self._batch_counter)))
+                cases.append(BatchedTestCase(self._batch_counter, real_kind, case_config, self.problem,
+                                             self._resolve_testcases(case_config['batched'], self._batch_counter,
+                                                                     kind=real_kind)))
             else:
-                cases.append(TestCase(self._testcase_counter, batch_no, case_config, self.problem))
+                cases.append(TestCase(self._testcase_counter, batch_no, real_kind, case_config, self.problem))
                 self._testcase_counter += 1
         return cases
 
     def cases(self):
+        if 'sample_test_cases' in self.problem.config:
+            samples = self._resolve_testcases(self.problem.config.sample_test_cases, kind=TestCase.KIND_SAMPLE)
+        else:
+            samples = []
+
         key = 'pretest_test_cases' if self.is_pretested else 'test_cases'
-        return self._resolve_testcases(self.problem.config[key])
+        kind = TestCase.KIND_PRETEST if self.is_pretested else TestCase.KIND_NORMAL
+        return samples + self._resolve_testcases(self.problem.config[key], kind=kind)

dmoj/judge.py

@@ -9,12 +9,12 @@
 from http.server import HTTPServer
 from itertools import chain
 
-from dmoj import graders, packet
+from dmoj import packet
 from dmoj.control import JudgeControlRequestHandler
 from dmoj.error import CompileError
 from dmoj.judgeenv import clear_problem_dirs_cache, env, get_supported_problems, startup_warnings
 from dmoj.monitor import DummyMonitor, Monitor
-from dmoj.problem import BatchedTestCase, Problem
+from dmoj.problem import BatchedTestCase, Problem, TestCase
 from dmoj.result import Result
 from dmoj.utils.ansi import ansi_style, print_ansi, strip_ansi
 from dmoj.utils.unicode import unicode_stdout_stderr, utf8bytes, utf8text
@@ -82,17 +82,8 @@ def update_problems(self):
         self.updater_signal.set()
 
     def _block_and_grade(self, problem, language, source, short_circuit, report=print):
-        if 'signature_grader' in problem.config:
-            grader_class = graders.SignatureGrader
-        elif 'interactive' in problem.config:
-            grader_class = graders.BridgedInteractiveGrader
-        elif 'custom_judge' in problem.config:
-            grader_class = graders.CustomGrader
-        else:
-            grader_class = graders.StandardGrader
-
         try:
-            self.current_grader = grader_class(self, problem, language, utf8bytes(source))
+            self.current_grader = problem.grader_class(self, problem, language, utf8bytes(source))
         except CompileError as compilation_error:
             error = compilation_error.args[0] or b'compiler exited abnormally'
 
@@ -187,24 +178,23 @@ def grade_cases(self, grader, cases, short_circuit=False, is_short_circuiting=Fa
             if isinstance(case, BatchedTestCase):
                 yield BatchBegin()
 
-                for batched_case in self.grade_cases(grader, case.batched_cases,
-                                                     short_circuit=case.config['short_circuit'],
-                                                     is_short_circuiting=is_short_circuiting):
+                for batched_result in self.grade_cases(grader, case.batched_cases,
+                                                       short_circuit=case.config['short_circuit'],
+                                                       is_short_circuiting=is_short_circuiting):
                     # A batched case just failed.
                     # There are two cases where this means that we should completely short-circuit:
                     # 1. If the batch was worth 0 points, to emulate the property of 0-point cases.
                     # 2. If the short_circuit flag is true, see <https://github.com/DMOJ/judge/issues/341>.
-                    if (batched_case.result_flag & Result.WA) and (not case.points or short_circuit):
+                    if (batched_result.result_flag & Result.WA) and \
+                            (short_circuit or case.kind in (TestCase.KIND_SAMPLE, TestCase.KIND_PRETEST)):
                         is_short_circuiting = True
-                    yield batched_case
+                    yield batched_result
                 yield BatchEnd()
                 continue
 
             # Stop grading if we're short circuiting
             if is_short_circuiting:
-                result = Result(case)
-                result.result_flag = Result.SC
-                yield result
+                yield Result(case, Result.SC)
                 continue
 
             result = grader.grade(case)
@@ -216,7 +206,8 @@ def grade_cases(self, grader, cases, short_circuit=False, is_short_circuiting=Fa
             # If the WA bit of result_flag is set and we are set to short-circuit (e.g., in a batch),
             # short circuit the rest of the cases.
             # Do the same if the case is a pretest (i.e. has 0 points)
-            if (result.result_flag & Result.WA) > 0 and (short_circuit or not case.points):
+            if (result.result_flag & Result.WA) > 0 and \
+                    (short_circuit or case.kind in (TestCase.KIND_SAMPLE, TestCase.KIND_PRETEST)):
                 is_short_circuiting = True
 
             yield result

dmoj/packet.py

@@ -184,6 +184,7 @@ def _flush_testcase_queue(self):
                                'cases': [
                                    {
                                        'position': position,
+                                       'kind': result.case.kind,
                                        'status': result.result_flag,
                                        'time': result.execution_time,
                                        'points': result.points,

dmoj/problem.py

@@ -165,6 +165,19 @@ def _resolve_archive_files(self):
             return archive
         return None
 
+    @property
+    def grader_class(self):
+        from dmoj import graders
+
+        if 'signature_grader' in self.config:
+            return graders.SignatureGrader
+        elif 'interactive' in self.config:
+            return graders.BridgedInteractiveGrader
+        elif 'custom_judge' in self.config:
+            return graders.CustomGrader
+        else:
+            return graders.StandardGrader
+
 
 class ProblemDataManager(dict):
     def __init__(self, problem_id, **kwargs):
@@ -190,9 +203,10 @@ def __del__(self):
 
 
 class BatchedTestCase:
-    def __init__(self, batch_no, config, problem, cases):
+    def __init__(self, batch_no, kind, config, problem, cases):
         self.config = config
         self.batch_no = batch_no
+        self.kind = kind
         self.points = config.points
         self.batched_cases = cases
         if any(isinstance(case, BatchedTestCase) for case in self.batched_cases):
@@ -204,9 +218,14 @@ def __str__(self):
 
 
 class TestCase:
-    def __init__(self, count, batch_no, config, problem):
+    KIND_NORMAL = 'normal'
+    KIND_PRETEST = 'pretest'
+    KIND_SAMPLE = 'sample'
+
+    def __init__(self, count, batch_no, kind, config, problem):
         self.position = count
         self.batch = batch_no
+        self.kind = kind
         self.config = config
         self.problem = problem
         self.points = config.points
@@ -344,4 +363,5 @@ def free_data(self):
         self._generated = None
 
     def __str__(self):
-        return 'TestCase{in=%s,out=%s,points=%s}' % (self.config['in'], self.config['out'], self.config['points'])
+        return 'TestCase{in=%s,out=%s,points=%s,kind=%s}' % (self.config['in'], self.config['out'],
+                                                             self.config['points'], self.kind)

dmoj/result.py

@@ -25,8 +25,8 @@ class Result:
         'IE': 'red',
     }
 
-    def __init__(self, case):
-        self.result_flag = 0
+    def __init__(self, case, result_flag=0):
+        self.result_flag = result_flag
         self.execution_time = 0
         self.wall_clock_time = 0
         self.max_memory = 0