script/benchstat: introduce interleave mode when running benchmark (#583

)

Currently, all iterations of the benchmark for old commit run before the one for
new commit. This can make the noise not evenly distributed across 2 benchmark
runs (e.g. the machine heats up too much after running all iterations the old
benchmark). This commit introduces interleave mode when running benchmark where
each iteration of benchmark for old commit is followed by an iteration of
benchmark for new one then back to an iteration of benchmark for old one.

.github/workflows/evm-benchmark.yml

@@ -44,9 +44,9 @@ jobs:
           go install golang.org/x/perf/cmd/[email protected]
           cd core/vm
           if [[ ${{ github.event_name }} = "pull_request" ]]; then
-            ../../script/benchstat.sh -c "go test -test.v -run=^$ -bench=BenchmarkEvm -count=10" \
-              -o $(git rev-parse --short origin/master) -n ${{ github.sha }}
+            ../../script/benchstat.sh -c "-test.v -test.run=^$ -test.bench=BenchmarkEvm" \
+              -o $(git rev-parse --short origin/master) -n ${{ github.sha }} -c 10 -i
           else
-            ../../script/benchstat.sh -c "go test -test.v -run=^$ -bench=BenchmarkEvm -count=10" \
-              -o "$OLD_COMMIT" -n "$NEW_COMMIT"
+            ../../script/benchstat.sh -c "-test.v -test.run=^$ -test.bench=BenchmarkEvm" \
+              -o "$OLD_COMMIT" -n "$NEW_COMMIT" -c 10 -i
           fi

script/benchstat.sh

@@ -3,24 +3,30 @@
 unset COMMAND
 unset OLD_COMMIT
 unset NEW_COMMIT
+unset COUNT
+unset INTERLEAVE
 PROGNAME=$(basename "$0")
-USAGE="$PROGNAME -c command -o old_commit -n new_commit
+USAGE="$PROGNAME -f flags -c count -o old_commit -n new_commit [-i]
 
 where:
-    command:     benchmark command
-    old_commit:  the old commit
-    new_commit:  the new commit
+    -f flags:       benchmark flags (e.g. -test.v -test.run=^$ -test.bench=BenchmarkGetAccount)
+    -o old_commit:  the old commit
+    -n new_commit:  the new commit
+    -c count:       the number of benchmark runs
+    -i:             interleave mode
 "
 
-while getopts :hc:o:n: option; do
+while getopts :hf:o:n:c:i option; do
   case $option in
     h)
       echo "$USAGE"
       exit 0
       ;;
-    c) COMMAND=$OPTARG ;;
+    f) COMMAND=$OPTARG ;;
     o) OLD_COMMIT=$OPTARG ;;
     n) NEW_COMMIT=$OPTARG ;;
+    c) COUNT=$OPTARG ;;
+    i) INTERLEAVE=true ;;
     :)
       echo "$PROGNAME: option requires an argument -- '$OPTARG'" >&2
       exit 1
@@ -34,30 +40,51 @@ while getopts :hc:o:n: option; do
 done
 shift $((OPTIND - 1))
 
-if [[ -z $COMMAND || -z $OLD_COMMIT || -z $NEW_COMMIT ]]; then
+if [[ -z $COMMAND || -z $OLD_COMMIT || -z $NEW_COMMIT || -z $COUNT ]]; then
   echo "$PROGNAME: missing mandatory option" >&2
   echo "Try '$(basename "$0") -h' for more information" >&2
   exit 1
 fi
 
 CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
 if [[ $OLD_COMMIT =~ "HEAD" ]]; then
-	OLD_COMMIT=$(git rev-parse $OLD_COMMIT)
+  OLD_COMMIT=$(git rev-parse $OLD_COMMIT)
 fi
 
 if [[ $NEW_COMMIT =~ "HEAD" ]]; then
-	NEW_COMMIT=$(git rev-parse $NEW_COMMIT)
+  NEW_COMMIT=$(git rev-parse $NEW_COMMIT)
 fi
 
 set -ex
 
-git checkout $OLD_COMMIT
-$COMMAND | tee old.txt
-git checkout $NEW_COMMIT
-$COMMAND | tee new.txt
+if [[ "$INTERLEAVE" = "true" ]]; then
+  git checkout $OLD_COMMIT
+  go test -c -o old.test
+  OLD_COMMAND="./old.test $COMMAND"
+
+  git checkout $NEW_COMMIT
+  go test -c -o new.test
+  NEW_COMMAND="./new.test $COMMAND"
+
+  for i in $(seq 1 $COUNT); do
+    $OLD_COMMAND | tee -a old.txt
+    $NEW_COMMAND | tee -a new.txt
+  done
+
+else
+  COMMAND="go test $COMMAND -test.count=$COUNT"
+  git checkout $OLD_COMMIT
+  $COMMAND | tee old.txt
+  git checkout $NEW_COMMIT
+  $COMMAND | tee new.txt
+fi
 
 benchstat old.txt new.txt
 
 git checkout $CURRENT_BRANCH
 rm old.txt
 rm new.txt
+
+if [[ "$INTERLEAVE" = "true" ]]; then
+  rm old.test new.test
+fi