src/hevm/run-consensus-tests

#!/usr/bin/env bash
set -e

# Invoke with hevm e.g.
# hevm compliance --tests ~/ethereum-tests --group VM --skip quadratic --html

HEVM=${HEVM:-hevm}

if [[ "$#" -lt 1 ]]; then
  echo >&2 "usage: $(basename "$0") <tests-dir>"
  exit 1
fi

tests=$1
html=$2
match=$3
skip=$4
timeout=${5:-10}

_html() {
cat <<.
<!doctype html>
<title>hevm test results</title>
<style>
* { font-family:
"latin modern mono", "fantasque sans mono",
inconsolata, menlo, monospace;
font-size: 22px;
line-height: 26px; }
body { margin: 2rem; }
header { text-align: center; margin: 4rem 0; }
table { border-collapse: collapse; width: 100%; }
tr:nth-child(even) { background: rgba(0, 0, 0, 0.05); }
td:not(:first-child):not(:last-child) { padding: 0 1rem; }
.category { opacity: 0.6; text-align: right }
a { color: darkblue; text-decoration: none; }
h1, h2 { text-align: center; margin-top: 2rem  }
.testcase { font-weight: bold }
#failed .testcase { color: rgb(200, 0, 0) }
#passed .testcase { color: rgb(0, 150, 0) }
#skipped .testcase { color: rgb(0, 100, 250) }
</style>
<header>
<h1>hevm consensus test report</h1>
<p>$(date +%Y-%m-%d)</p>
<p>$(echo "$npass passed, $nfail failed, $nskip skipped")</p>
(Test suite: <span class=VMTests</span>VMTests</span> for ConstantinopleFix)
</header>
<h2>Failed tests</h2>
<table id=failed>
<tbody>
$(echo $failed)
</table>
<h2>Skipped tests</h2>
<table id=skipped>
<tbody>
$(echo $skipped)
</table>
<h2>Passed tests</h2>
<table id=passed>
<tbody>
$(echo $passed)
</table>
.
}

{
  cd "$tests"
  for x in VMTests/*/*; do
    if [[ $x =~ .*$match.* ]] && [[ -n $skip && $x =~ .*$skip.* ]]; then
      for job in $(<$x jq '.|keys[]' -r); do
        echo "$x $job skip"
      done
    elif [[ $x =~ .*$match.* ]]; then
      echo -n "$x " ; "$HEVM" vm-test --file $x --timeout $timeout 2>&1
    fi
  done
} | {
  while read path test outcome; do
    echo >&2 "$path $test $outcome"
    category=$(dirname "$path")
    testcase=$(basename "${path%.json}")
    row="<tr><td class=testcase>$testcase<td>$outcome<td class=category>$category"
    row+=$'\n'
    case $outcome in
      ok)      passed+=$row ;;
      skip)    skipped+=$row ;;
      timeout) timouts+=row ;;
      *)       failed+=$row ;;
    esac
  done

  sum () { echo -ne "$1" | wc -l | awk '{print $1}'; }

  nfail=$(sum "$failed")
  npass=$(sum "$passed")
  nskip=$(sum "$skipped")
  ntime=$(sum "$timeouts")

  echo >&2 "passed: $npass"
  echo >&2 "failed: $nfail"
  echo >&2 "timeout: $ntime"
  echo >&2 "skipped: $nskip"

  if [[ $html == "True" ]]; then
    _html
  fi

  [[ $nfail -gt 0 ]] && exit 1 || exit 0
}