Added Meta test that subtracts points #209 (#266)

* remove upstream

* introducing penalty metachecks

* Added penalty meta test exectution step

* added penalty meta tests to final output writer

* added logging for penalty metatest

* fixed equals in exec step

* suitable Result constructor in testutils

* Penalty MetaTests integration Tests _1

* Update README with penalty meta tests description

* More tests on PenaltyMetaTests

* Increase consistency in writing the result

* improve conditions about displaying metatests and codechecks results

* Standard result writer tests

* penalty meta tests in Weblab result writer

* Remove unnecessary import

* Fix logic bug

* Revert "Increase consistency in writing the result"

This reverts commit bc24b9c.

* Revert output format changes

---------

Co-authored-by: puctom <[email protected]>
Co-authored-by: Martin Mladenov <[email protected]>

README.md

@@ -12,6 +12,7 @@
  * [Code coverage](#code-coverage)
  * [Mutation coverage](#mutation-coverage)
  * [Meta tests](#meta-tests)
+ * [Penalty meta tests](#penalty-meta-tests)
  * [Code checks](#code-checks)
  * [Penalty code checks](#penalty-code-checks)
  * [Success message](#success-message)
@@ -285,6 +286,39 @@ Meta test: returns wrong elements (weight: 1) FAILED
 Meta test: compares only elements in the same index (weight: 3) FAILED
 Meta test: background service does not work (weight: 2) PASSED
 ```
+#### Penalty meta tests
+
+Penalty meta tests work in a similar way to regular meta tests. However, they are not considered a grading component. Instead, if a penalty meta test passes, this has no effect on the final score. However, if it fails, its weight is subtracted from the final grade.
+
+Penalty meta tests must have a positive weight, but there is no upper limit on their total weight. For example, if there are two penalty meta tests where one of them has a weight of 10 and the other one 100 (see the example below), and a student fails both of them, the total applied penalty would be 110. In case this makes the final grade negative, the grade is reported as 0 instead.
+
+Penalty meta tests can be defined as follows:
+
+```java
+@Override
+public List<MetaTest> penaltyMetaTests() {
+ return List.of(
+ MetaTest.insertAt(2, "returns empty list when list1 is null", 29,
+ "if (list1 == null) return result;"
+ ),
+ MetaTest.withStringReplacement("returns wrong elements",
+ "if (hashSet.contains(e))",
+ "if (!hashSet.contains(e))"
+ ),
+ MetaTest.withLineReplacement(3, "compares only elements in the same index", 29, 41,
+ """
+ for (int i = 0; i < list1.size() && i < list2.size(); i++) {
+ if (list1.get(i).equals(list2.get(i))) {
+ result.add(list1.get(i));
+ }
+ }
+ """
+ )
+ );
+}
+```
+
+If this method is not overridden, penalty meta tests are disabled.
 
 #### Code checks
 

andy/src/main/java/nl/tudelft/cse1110/andy/config/RunConfiguration.java

@@ -45,6 +45,10 @@ public List<MetaTest> metaTests() {
  return Collections.emptyList();
  }
 
+ public List<MetaTest> penaltyMetaTests() {
+ return Collections.emptyList();
+ }
+
  public List<String> listOfMutants() {
  return DEFAULTS;
  }

andy/src/main/java/nl/tudelft/cse1110/andy/execution/mode/ModeActionSelector.java

@@ -129,6 +129,7 @@ public static List<ExecutionStep> fullMode() {
  new RunPenaltyCodeChecksStep(),
  new RunCodeChecksStep(),
  new KillExternalProcessStep(),
+ new RunPenaltyMetaTestsStep(),
  new RunMetaTestsStep()
  );
  }

andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunPenaltyMetaTestsStep.java

@@ -0,0 +1,62 @@
+package nl.tudelft.cse1110.andy.execution.step;
+
+import nl.tudelft.cse1110.andy.config.DirectoryConfiguration;
+import nl.tudelft.cse1110.andy.config.MetaTest;
+import nl.tudelft.cse1110.andy.config.RunConfiguration;
+import nl.tudelft.cse1110.andy.execution.Context.Context;
+import nl.tudelft.cse1110.andy.execution.ExecutionStep;
+import nl.tudelft.cse1110.andy.result.MetaTestResult;
+import nl.tudelft.cse1110.andy.result.ResultBuilder;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class RunPenaltyMetaTestsStep implements ExecutionStep {
+
+ @Override
+ public void execute(Context ctx, ResultBuilder result) {
+ DirectoryConfiguration dirCfg = ctx.getDirectoryConfiguration();
+ RunConfiguration runCfg = ctx.getRunConfiguration();
+
+ int score = 0;
+ int totalWeight = 0;
+
+ ClassLoader currentClassLoader = Thread.currentThread().getContextClassLoader();
+
+ try {
+
+ List<MetaTest> metaTests = runCfg.penaltyMetaTests();
+ List<MetaTestResult> metaTestResults = new ArrayList<>();
+
+ for (MetaTest metaTest : metaTests) {
+
+ boolean passesTheMetaTest = metaTest.execute(ctx, dirCfg, runCfg);
+
+ if (passesTheMetaTest) {
+ score += metaTest.getWeight();
+ }
+
+ metaTestResults.add(new MetaTestResult(metaTest.getName(), metaTest.getWeight(), passesTheMetaTest));
+
+ totalWeight += metaTest.getWeight();
+ }
+
+ result.logPenaltyMetaTests(score, totalWeight, metaTestResults);
+ } catch (Exception ex) {
+ result.genericFailure(this, ex);
+ } finally {
+ /* restore the class loader to the one before meta tests */
+ Thread.currentThread().setContextClassLoader(currentClassLoader);
+ }
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ return other instanceof RunPenaltyMetaTestsStep;
+ }
+
+ @Override
+ public int hashCode() {
+ return super.hashCode();
+ }
+}

andy/src/main/java/nl/tudelft/cse1110/andy/grade/GradeValues.java

@@ -86,15 +86,16 @@ public GradeValues setPenalty(int penalty) {
  return this;
  }
 
- public static GradeValues fromResults(CoverageResult coverageResults, CodeChecksResult codeCheckResults, MutationTestingResult mutationResults, MetaTestsResult metaTestResults, CodeChecksResult penaltyCodeCheckResults) {
+ public static GradeValues fromResults(CoverageResult coverageResults, CodeChecksResult codeCheckResults, MutationTestingResult mutationResults, MetaTestsResult metaTestResults, MetaTestsResult penaltyMetaTestResults, CodeChecksResult penaltyCodeCheckResults) {
  GradeValues grades = new GradeValues();
  grades.setBranchGrade(coverageResults.getCoveredBranches(), coverageResults.getTotalNumberOfBranches());
  grades.setCheckGrade(codeCheckResults.getNumberOfPassedChecks(), codeCheckResults.getTotalNumberOfChecks());
  grades.setMutationGrade(mutationResults.getKilledMutants(), mutationResults.getTotalNumberOfMutants());
  grades.setMetaGrade(metaTestResults.getPassedMetaTests(), metaTestResults.getTotalTests());
 
  // penalty is equal to the sum of the weights of all failed penalty code checks
- grades.setPenalty(penaltyCodeCheckResults.getCheckResults().stream().mapToInt(check -> check.passed() ? 0 : check.getWeight()).sum());
+ grades.setPenalty(penaltyCodeCheckResults.getCheckResults().stream().mapToInt(check -> check.passed() ? 0 : check.getWeight()).sum()
+ + penaltyMetaTestResults.getMetaTestResults().stream().mapToInt(check -> check.succeeded() ? 0 : check.getWeight()).sum());
 
  return grades;
  }

andy/src/main/java/nl/tudelft/cse1110/andy/result/Result.java

@@ -13,21 +13,23 @@ public class Result {
  private final CodeChecksResult penaltyCodeChecks;
  private final CoverageResult coverage;
  private final MetaTestsResult metaTests;
+ private final MetaTestsResult penaltyMetaTests;
  private final int penalty;
  private final int finalGrade;
  private final GenericFailure genericFailure;
  private final double timeInSeconds;
  private final GradeWeight weights;
  private final String successMessage;
 
- public Result(CompilationResult compilation, UnitTestsResult tests, MutationTestingResult mutationTesting, CodeChecksResult codeChecks, CodeChecksResult penaltyCodeChecks, CoverageResult coverage, MetaTestsResult metaTests, int penalty, int finalGrade, GenericFailure genericFailure, double timeInSeconds, GradeWeight weights, String successMessage) {
+ public Result(CompilationResult compilation, UnitTestsResult tests, MutationTestingResult mutationTesting, CodeChecksResult codeChecks, CodeChecksResult penaltyCodeChecks, CoverageResult coverage, MetaTestsResult metaTests, MetaTestsResult penaltyMetaTests, int penalty, int finalGrade, GenericFailure genericFailure, double timeInSeconds, GradeWeight weights, String successMessage) {
  this.compilation = compilation;
  this.tests = tests;
  this.mutationTesting = mutationTesting;
  this.codeChecks = codeChecks;
  this.penaltyCodeChecks = penaltyCodeChecks;
  this.coverage = coverage;
  this.metaTests = metaTests;
+ this.penaltyMetaTests = penaltyMetaTests;
  this.penalty = penalty;
  this.finalGrade = finalGrade;
  this.genericFailure = genericFailure;
@@ -40,7 +42,7 @@ public Result(CompilationResult compilation, UnitTestsResult tests, MutationTest
  }
 
  public Result(CompilationResult compilation, double timeInSeconds) {
- this(compilation, UnitTestsResult.empty(), MutationTestingResult.empty(), CodeChecksResult.empty(), CodeChecksResult.empty(), CoverageResult.empty(), MetaTestsResult.empty(), 0, 0, GenericFailure.noFailure(), timeInSeconds, null, null);
+ this(compilation, UnitTestsResult.empty(), MutationTestingResult.empty(), CodeChecksResult.empty(), CodeChecksResult.empty(), CoverageResult.empty(), MetaTestsResult.empty(), MetaTestsResult.empty(), 0, 0, GenericFailure.noFailure(), timeInSeconds, null, null);
  }
 
  public CompilationResult getCompilation() {
@@ -70,6 +72,9 @@ public CoverageResult getCoverage() {
  public MetaTestsResult getMetaTests() {
  return metaTests;
  }
+ public MetaTestsResult getPenaltyMetaTests() {
+ return penaltyMetaTests;
+ }
 
  public int getPenalty() {
  return penalty;

andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java

@@ -49,6 +49,7 @@ public class ResultBuilder {
  private CodeChecksResult penaltyCodeCheckResults = CodeChecksResult.empty();
  private CoverageResult coverageResults = CoverageResult.empty();
  private MetaTestsResult metaTestResults = MetaTestsResult.empty();
+ private MetaTestsResult penaltyMetaTestResults = MetaTestsResult.empty();
 
  public ResultBuilder(Context ctx, GradeCalculator gradeCalculator) {
  this.ctx = ctx;
@@ -244,6 +245,10 @@ public void logMetaTests(int score, int totalTests, List<MetaTestResult> metaTes
  this.metaTestResults.addResults(score, totalTests, metaTestResults);
  }
 
+ public void logPenaltyMetaTests(int score, int totalTests, List<MetaTestResult> metaTestResults) {
+ this.penaltyMetaTestResults.addResults(score, totalTests, metaTestResults);
+ }
+
  /*
  * Generic failures
  */
@@ -271,7 +276,7 @@ public Result build() {
  if(!compilation.successful()) {
  return new Result(compilation, timeInSeconds);
  } else {
- GradeValues grades = GradeValues.fromResults(coverageResults, codeCheckResults, mutationResults, metaTestResults, penaltyCodeCheckResults);
+ GradeValues grades = GradeValues.fromResults(coverageResults, codeCheckResults, mutationResults, metaTestResults, penaltyMetaTestResults, penaltyCodeCheckResults);
  GradeWeight weights = GradeWeight.fromConfig(ctx.getRunConfiguration().weights());
  String successMessage = ctx.getRunConfiguration().successMessage();
 
@@ -287,6 +292,7 @@ public Result build() {
  penaltyCodeCheckResults,
  coverageResults,
  metaTestResults,
+ penaltyMetaTestResults,
  penalty,
  finalGrade,
  genericFailureObject,

andy/src/main/java/nl/tudelft/cse1110/andy/writer/standard/StandardResultWriter.java

@@ -65,7 +65,7 @@ private void writeStdOutFile(Context ctx, Result result) {
  printCoverageResults(result.getCoverage());
  printMutationTestingResults(result.getMutationTesting());
  printCodeCheckResults(ctx, result.getCodeChecks(), result.getPenaltyCodeChecks());
- printMetaTestResults(ctx, result.getMetaTests());
+ printMetaTestResults(ctx, result.getMetaTests(), result.getPenaltyMetaTests());
  printFinalGrade(ctx, result);
  printModeAndTimeToRun(ctx, result.getTimeInSeconds());
  }
@@ -244,8 +244,8 @@ private void printModeAndTimeToRun(Context ctx, double timeInSeconds) {
  l(String.format("\nAndy took %.1f seconds to assess your solution.", timeInSeconds));
  }
 
- private void printMetaTestResults(Context ctx, MetaTestsResult metaTests) {
- if (!metaTests.wasExecuted() || metaTests.hasNoMetaTests())
+ private void printMetaTestResults(Context ctx, MetaTestsResult metaTests, MetaTestsResult penaltyMetaTests) {
+ if ((!metaTests.wasExecuted() || metaTests.hasNoMetaTests()) && (!penaltyMetaTests.wasExecuted() || penaltyMetaTests.hasNoMetaTests()))
  return;
 
  boolean allHints = modeActionSelector(ctx).shouldShowFullHints();
@@ -255,7 +255,8 @@ private void printMetaTestResults(Context ctx, MetaTestsResult metaTests) {
  return;
 
  l("\n--- Meta tests");
- l(String.format("%d/%d passed", metaTests.getPassedMetaTests(), metaTests.getTotalTests()));
+ l(String.format("%d/%d passed", metaTests.getPassedMetaTests() + penaltyMetaTests.getPassedMetaTests(),
+ metaTests.getTotalTests() + penaltyMetaTests.getTotalTests()));
 
  if (allHints) {
  for (MetaTestResult metaTestResult : metaTests.getMetaTestResults()) {
@@ -265,6 +266,13 @@ private void printMetaTestResults(Context ctx, MetaTestsResult metaTests) {
  l(String.format("Meta test: %s (weight: %d) FAILED", metaTestResult.getName(), metaTestResult.getWeight()));
  }
  }
+ for (MetaTestResult penaltyResult : penaltyMetaTests.getMetaTestResults()) {
+ if (penaltyResult.succeeded()) {
+ l(String.format("Meta test: %s (penalty: %d) PASSED", penaltyResult.getName(), penaltyResult.getWeight()));
+ } else {
+ l(String.format("Meta test: %s (penalty: %d) FAILED", penaltyResult.getName(), penaltyResult.getWeight()));
+ }
+ }
  }
 
  }

andy/src/test/java/integration/LibraryMetaTestsTest.java

@@ -88,4 +88,46 @@ void metaTestInternalFailure() {
  .isEqualTo(RunMetaTestsStep.class.getSimpleName());
  }
 
+ @Test
+ void allPenaltyMetaTestsPassing() {
+ Result result = run("NumberUtilsAddLibrary",
+ "NumberUtilsAddOfficialSolution", "NumberUtilsAddWithPenaltyMetaTestsConfiguration");
+
+ assertThat(result.getPenaltyMetaTests().getTotalTests()).isEqualTo(3);
+ assertThat(result.getPenaltyMetaTests().getPassedMetaTests()).isEqualTo(3);
+ assertThat(result.getPenaltyMetaTests())
+ .has(passedMetaTest("DoesNotApplyCarryAtAll"))
+ .has(passedMetaTest("DoesNotApplyLastCarry"))
+ .has(passedMetaTest("DoesNotCheckNumbersOutOfRange"));
+ }
+
+ @Test
+ void somePenaltyMetaTestFailing() {
+ Result result = run("NumberUtilsAddLibrary",
+ "NumberUtilsAddAllTestsPass", "NumberUtilsAddWithPenaltyMetaTestsConfiguration");
+
+ assertThat(result.getPenaltyMetaTests().getTotalTests()).isEqualTo(3);
+ assertThat(result.getPenaltyMetaTests().getPassedMetaTests()).isEqualTo(1);
+
+ assertThat(result.getPenaltyMetaTests())
+ .has(passedMetaTest("DoesNotCheckNumbersOutOfRange"))
+ .has(failedMetaTest("DoesNotApplyCarryAtAll"))
+ .has(failedMetaTest("DoesNotApplyLastCarry"));
+ }
+
+ @Test
+ void somePenaltyMetaTestFailingWithWeights() {
+ Result result = run("NumberUtilsAddLibrary",
+ "NumberUtilsAddAllTestsPass", "NumberUtilsAddConfigurationWithWeightWithPenaltyMetaTests");
+
+ assertThat(result.getPenaltyMetaTests().getTotalTests()).isEqualTo(7);
+ assertThat(result.getPenaltyMetaTests().getPassedMetaTests()).isEqualTo(2);
+
+ assertThat(result.getPenaltyMetaTests())
+ .has(passedMetaTest("DoesNotCheckNumbersOutOfRange"))
+ .has(failedMetaTest("AppliesMultipleCarriesWrongly"))
+ .has(failedMetaTest("DoesNotApplyCarryAtAll"))
+ .has(failedMetaTest("DoesNotApplyLastCarry"));
+ }
+
 }